From a9b21c87835ae10f438332123238de2ebf46652f Mon Sep 17 00:00:00 2001 From: Eddie Date: Thu, 12 Dec 2024 19:38:12 -0300 Subject: [PATCH 1/7] Save/List/Delete assets for tool --- .../src/network/handle_commands_list.rs | 43 ++++ .../network/v2_api/api_v2_commands_tools.rs | 108 ++++++++ .../llm_language_support/file_support_py.rs | 20 +- .../llm_language_support/file_support_ts.rs | 86 ++++--- .../src/api_v2/api_v2_handlers_tools.rs | 236 +++++++++++++++++- .../shinkai-http-api/src/node_commands.rs | 21 ++ .../src/tools/deno_tools.rs | 24 +- .../src/tools/python_tools.rs | 24 +- 8 files changed, 509 insertions(+), 53 deletions(-) diff --git a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs index e38dacc94..1e0591b9f 100644 --- a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs +++ b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs @@ -2736,6 +2736,49 @@ impl Node { let _ = Node::v2_api_set_oauth_token(db_clone, bearer, code, state, res).await; }); } + NodeCommand::V2ApiUploadToolAsset { + bearer, + tool_id, + app_id, + file_name, + file_data, + res, + } => { + let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); + tokio::spawn(async move { + let _ = Node::v2_api_upload_tool_asset( + db_clone, bearer, tool_id, app_id, file_name, file_data, node_env, res, + ) + .await; + }); + } + NodeCommand::V2ApiListToolAssets { + bearer, + tool_id, + app_id, + res, + } => { + let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); + tokio::spawn(async move { + let _ = Node::v2_api_list_tool_assets(db_clone, bearer, tool_id, app_id, node_env, res).await; + }); + } + NodeCommand::V2ApiDeleteToolAsset { + bearer, + tool_id, + app_id, + file_name, + res, + } => { + let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); + tokio::spawn(async move { + let _ = Node::v2_api_delete_tool_asset(db_clone, bearer, tool_id, app_id, file_name, node_env, res) + .await; + }); + } _ => (), } } diff --git a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs index a5a20f0aa..b4880ffa0 100644 --- a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs +++ b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs @@ -8,6 +8,7 @@ use crate::{ tool_generation::v2_create_and_send_job_message, tool_prompts::{generate_code_prompt, tool_metadata_implementation_prompt}, }, + utils::environment::NodeEnvironment, }; use std::io::Read; @@ -1738,6 +1739,113 @@ impl Node { Err(err) => Err(format!("Failed to update job config: {}", err)), } } + + pub async fn v2_api_upload_tool_asset( + db: Arc>, + bearer: String, + _tool_id: String, + app_id: String, + file_name: String, + file_data: Vec, + node_env: NodeEnvironment, + res: Sender>, + ) -> Result<(), NodeError> { + // Validate the bearer token + if Self::validate_bearer_token(&bearer, db.clone(), &res).await.is_err() { + return Ok(()); + } + + let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); + file_path.push(".tools_storage"); + file_path.push(app_id); + // TODO + // keep this in sync with deno/python runner + file_path.push("assets"); + // Create directories if they don't exist + if !file_path.exists() { + std::fs::create_dir_all(&file_path)?; + } + file_path.push(&file_name); + std::fs::write(&file_path, &file_data)?; + + let response = json!({ + "status": "success", + "message": "Tool asset uploaded successfully", + "file": file_data.len(), + "file_name": file_name + }); + let _ = res.send(Ok(response)).await; + Ok(()) + } + + pub async fn v2_api_list_tool_assets( + db: Arc>, + bearer: String, + tool_id: String, + app_id: String, + node_env: NodeEnvironment, + res: Sender, APIError>>, + ) -> Result<(), NodeError> { + // Validate the bearer token + if Self::validate_bearer_token(&bearer, db.clone(), &res).await.is_err() { + return Ok(()); + } + + let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); + file_path.push(".tools_storage"); + file_path.push(app_id); + // TODO + // keep this in sync with deno/python runner + file_path.push("assets"); + let files = std::fs::read_dir(&file_path).unwrap(); + let file_names = files + .map(|file| file.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + let _ = res.send(Ok(file_names)).await; + Ok(()) + } + + pub async fn v2_api_delete_tool_asset( + db: Arc>, + bearer: String, + tool_id: String, + app_id: String, + file_name: String, + node_env: NodeEnvironment, + res: Sender>, + ) -> Result<(), NodeError> { + // Validate the bearer token + if Self::validate_bearer_token(&bearer, db.clone(), &res).await.is_err() { + return Ok(()); + } + + let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); + file_path.push(".tools_storage"); + file_path.push(app_id); + // TODO + // keep this in sync with deno/python runner + file_path.push("assets"); + file_path.push(&file_name); + let stat = std::fs::remove_file(&file_path).map_err(|err| APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Failed to delete file".to_string(), + message: format!("Failed to delete file: {}", err), + }); + match stat { + Ok(_) => { + let response = json!({ + "status": "success", + "message": "Tool asset deleted successfully", + "file_name": file_name + }); + let _ = res.send(Ok(response)).await; + } + Err(err) => { + let _ = res.send(Err(err)).await; + } + } + Ok(()) + } } #[cfg(test)] diff --git a/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_py.rs b/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_py.rs index 8b02f4f20..0c0d97bbb 100644 --- a/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_py.rs +++ b/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_py.rs @@ -5,16 +5,22 @@ pub fn generate_file_support_py(declaration_only: bool) -> String { "Gets an array of mounted files.", "List[str]", vec![], - "mount_paths = os.environ.get('SHINKAI_MOUNT')\n if not mount_paths:\n return []\n return [path.strip() for path in mount_paths.split(',')]", - "Array of files" + r#"mount_paths = os.environ.get('SHINKAI_MOUNT') + if not mount_paths: + return [] + return [path.strip() for path in mount_paths.split(',') if path.strip()]"#, + "Array of files", ), ( "get_asset_paths", "Gets an array of asset files. These files are read only.", "List[str]", vec![], - "asset_paths = os.environ.get('SHINKAI_ASSETS')\n if not asset_paths:\n return []\n return [path.strip() for path in asset_paths.split(',')]", - "Array of files" + r#"asset_paths = os.environ.get('SHINKAI_ASSETS') + if not asset_paths: + return [] + return [path.strip() for path in asset_paths.split(',') if path.strip()]"#, + "Array of files", ), ( "get_home_path", @@ -22,7 +28,7 @@ pub fn generate_file_support_py(declaration_only: bool) -> String { "str", vec![], "return os.environ.get('SHINKAI_HOME', '')", - "Home directory path" + "Home directory path", ), ( "get_shinkai_node_location", @@ -30,7 +36,7 @@ pub fn generate_file_support_py(declaration_only: bool) -> String { "str", vec![], "return os.environ.get('SHINKAI_NODE_LOCATION', '')", - "Shinkai Node URL" + "Shinkai Node URL", ), ( "get_access_token", @@ -54,7 +60,7 @@ pub fn generate_file_support_py(declaration_only: bool) -> String { except Exception as e: print(f'Error getting access token: {str(e)}') return ''"#, - "OAuth access token" + "OAuth access token", ), ]; diff --git a/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_ts.rs b/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_ts.rs index ad92da602..a1fefb077 100644 --- a/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_ts.rs +++ b/shinkai-bin/shinkai-node/src/tools/llm_language_support/file_support_ts.rs @@ -1,43 +1,47 @@ pub fn generate_file_support_ts(declaration_only: bool) -> String { let function_definitions = vec![ - ( - "getMountPaths", - "Gets an array of mounted files.", - "Promise", - vec![], - "const mountPaths = Deno.env.get('SHINKAI_MOUNT');\n if (!mountPaths) return [];\n return mountPaths.split(',').map(path => path.trim());", - "Array of files" - ), - ( - "getAssetPaths", - "Gets an array of asset files. These files are read only.", - "Promise", - vec![], - "const assetPaths = Deno.env.get('SHINKAI_ASSETS');\n if (!assetPaths) return [];\n return assetPaths.split(',').map(path => path.trim());", - "Array of files" - ), - ( - "getHomePath", - "Gets the home directory path. All created files must be written to this directory.", - "Promise", - vec![], - "return Deno.env.get('SHINKAI_HOME') || \"\";", - "Home directory path" - ), - ( - "getShinkaiNodeLocation", - "Gets the Shinkai Node location URL. This is the URL of the Shinkai Node server.", - "Promise", - vec![], - "return Deno.env.get('SHINKAI_NODE_LOCATION') || \"\";", - "Shinkai Node URL" - ), - ( - "getAccessToken", - "Gets a valid OAuth AccessToken for the given provider.", - "Promise", - vec!["providerName: string"], - r#" + ( + "getMountPaths", + "Gets an array of mounted files.", + "Promise", + vec![], + r#"const mountPaths = Deno.env.get('SHINKAI_MOUNT'); + if (!mountPaths) return []; + return mountPaths.split(',').map(path => path.trim()).filter(path => path.length > 0);"#, + "Array of files", + ), + ( + "getAssetPaths", + "Gets an array of asset files. These files are read only.", + "Promise", + vec![], + r#"const assetPaths = Deno.env.get('SHINKAI_ASSETS'); + if (!assetPaths) return []; + return assetPaths.split(',').map(path => path.trim()).filter(path => path.length > 0);"#, + "Array of files", + ), + ( + "getHomePath", + "Gets the home directory path. All created files must be written to this directory.", + "Promise", + vec![], + "return Deno.env.get('SHINKAI_HOME') || \"\";", + "Home directory path", + ), + ( + "getShinkaiNodeLocation", + "Gets the Shinkai Node location URL. This is the URL of the Shinkai Node server.", + "Promise", + vec![], + "return Deno.env.get('SHINKAI_NODE_LOCATION') || \"\";", + "Shinkai Node URL", + ), + ( + "getAccessToken", + "Gets a valid OAuth AccessToken for the given provider.", + "Promise", + vec!["providerName: string"], + r#" type ProviderConfig = { name: string, version: string, @@ -61,9 +65,9 @@ pub fn generate_file_support_ts(declaration_only: bool) -> String { } return providerConfig.accessToken || ''; "#, - "OAuth access token" - ), - ]; + "OAuth access token", + ), + ]; let mut output = String::new(); diff --git a/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs b/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs index 7fae2daaa..e6fcce541 100644 --- a/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs +++ b/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs @@ -7,6 +7,10 @@ use utoipa::{OpenApi, ToSchema}; use warp::Filter; use reqwest::StatusCode; use std::collections::HashMap; +use bytes::Bytes; +use futures::TryStreamExt; +use warp::multipart::{FormData, Part}; +use bytes::Buf; use crate::{node_api_router::APIError, node_commands::NodeCommand}; use super::api_v2_router::{create_success_response, with_sender}; @@ -157,6 +161,32 @@ pub fn tool_routes( .and(warp::body::json()) .and_then(import_tool_handler); + let tool_asset_route = warp::path("tool_asset") + .and(warp::post()) + .and(with_sender(node_commands_sender.clone())) + .and(warp::header::("authorization")) + .and(warp::header::("x-shinkai-tool-id")) + .and(warp::header::("x-shinkai-app-id")) + .and(warp::multipart::form()) + .and_then(tool_asset_handler); + + let list_tool_asset_route = warp::path("list_tool_asset") + .and(warp::get()) + .and(with_sender(node_commands_sender.clone())) + .and(warp::header::("authorization")) + .and(warp::header::("x-shinkai-tool-id")) + .and(warp::header::("x-shinkai-app-id")) + .and_then(list_tool_asset_handler); + + let delete_tool_asset_route = warp::path("tool_asset") + .and(warp::delete()) + .and(with_sender(node_commands_sender.clone())) + .and(warp::header::("authorization")) + .and(warp::header::("x-shinkai-tool-id")) + .and(warp::header::("x-shinkai-app-id")) + .and(warp::query::>()) + .and_then(delete_tool_asset_handler); + tool_execution_route .or(code_execution_route) .or(tool_definitions_route) @@ -177,6 +207,9 @@ pub fn tool_routes( .or(resolve_shinkai_file_protocol_route) .or(export_tool_route) .or(import_tool_route) + .or(tool_asset_route) + .or(list_tool_asset_route) + .or(delete_tool_asset_route) } #[utoipa::path( @@ -335,7 +368,7 @@ pub struct ToolMetadata { pub parameters: Value, } -#[derive(Deserialize, ToSchema)] +#[derive(Deserialize, ToSchema, Debug)] pub struct ToolImplementationRequest { pub message: JobMessage, pub language: CodeLanguage, @@ -1297,3 +1330,204 @@ pub async fn resolve_shinkai_file_protocol_handler( ) )] pub struct ToolsApiDoc; + +#[utoipa::path( + post, + path = "/v2/tool_asset", + responses( + (status = 200, description = "Successfully uploaded tool asset", body = Value), + (status = 400, description = "Bad request", body = APIError), + (status = 500, description = "Internal server error", body = APIError) + ) +)] +pub async fn tool_asset_handler( + sender: Sender, + authorization: String, + tool_id: String, + app_id: String, + mut form: FormData, +) -> Result { + let bearer = authorization.strip_prefix("Bearer ").unwrap_or("").to_string(); + + let mut file_name = String::new(); + let mut file_data: Option> = None; + + while let Ok(Some(part)) = form.try_next().await { + match part.name() { + "file_name" => { + // Convert the part to bytes then to string + let mut bytes = Vec::new(); + let mut stream = part.stream(); + while let Ok(Some(chunk)) = stream.try_next().await { + bytes.extend_from_slice(chunk.chunk()); + } + file_name = String::from_utf8_lossy(&bytes).into_owned(); + } + "file" => { + // Read file data + let mut bytes = Vec::new(); + let mut stream = part.stream(); + while let Ok(Some(chunk)) = stream.try_next().await { + bytes.extend_from_slice(chunk.chunk()); + } + file_data = Some(bytes); + } + _ => {} + } + } + + // Validate we have both file name and data + let file_data = match file_data { + Some(data) => data, + None => { + return Ok(warp::reply::with_status( + warp::reply::json(&APIError { + code: 400, + error: "Missing file".to_string(), + message: "File data is required".to_string(), + }), + StatusCode::BAD_REQUEST, + )) + } + }; + + if file_name.is_empty() { + return Ok(warp::reply::with_status( + warp::reply::json(&APIError { + code: 400, + error: "Missing file name".to_string(), + message: "File name is required".to_string(), + }), + StatusCode::BAD_REQUEST, + )); + } + + let (res_sender, res_receiver) = async_channel::bounded(1); + + sender + .send(NodeCommand::V2ApiUploadToolAsset { + bearer, + tool_id, + app_id, + file_name, + file_data, + res: res_sender, + }) + .await + .map_err(|_| warp::reject::reject())?; + + let result = res_receiver.recv().await.map_err(|_| warp::reject::reject())?; + + match result { + Ok(response) => { + let response = create_success_response(response); + Ok(warp::reply::with_status(warp::reply::json(&response), StatusCode::OK)) + } + Err(error) => Ok(warp::reply::with_status( + warp::reply::json(&error), + StatusCode::from_u16(error.code).unwrap(), + )), + } +} + +#[utoipa::path( + get, + path = "/v2/list_tool_asset", + responses( + (status = 200, description = "Successfully listed tool assets", body = Vec), + (status = 400, description = "Bad request", body = APIError), + (status = 500, description = "Internal server error", body = APIError) + ) +)] +pub async fn list_tool_asset_handler( + sender: Sender, + authorization: String, + tool_id: String, + app_id: String, +) -> Result { + let bearer = authorization.strip_prefix("Bearer ").unwrap_or("").to_string(); + + let (res_sender, res_receiver) = async_channel::bounded(1); + + sender + .send(NodeCommand::V2ApiListToolAssets { + bearer, + tool_id, + app_id, + res: res_sender, + }) + .await + .map_err(|_| warp::reject::reject())?; + + let result = res_receiver.recv().await.map_err(|_| warp::reject::reject())?; + + match result { + Ok(response) => { + let response = create_success_response(response); + Ok(warp::reply::with_status(warp::reply::json(&response), StatusCode::OK)) + } + Err(error) => Ok(warp::reply::with_status( + warp::reply::json(&error), + StatusCode::from_u16(error.code).unwrap(), + )), + } +} + +#[utoipa::path( + delete, + path = "/v2/tool_asset", + params( + ("file_name" = String, Query, description = "Name of the file to delete") + ), + responses( + (status = 200, description = "Successfully deleted tool asset", body = Value), + (status = 400, description = "Bad request", body = APIError), + (status = 500, description = "Internal server error", body = APIError) + ) +)] +pub async fn delete_tool_asset_handler( + sender: Sender, + authorization: String, + tool_id: String, + app_id: String, + query_params: HashMap, +) -> Result { + let bearer = authorization.strip_prefix("Bearer ").unwrap_or("").to_string(); + + let file_name = query_params + .get("file_name") + .ok_or_else(|| { + warp::reject::custom(APIError { + code: 400, + error: "Missing file name".to_string(), + message: "File name is required".to_string(), + }) + })? + .to_string(); + + let (res_sender, res_receiver) = async_channel::bounded(1); + + sender + .send(NodeCommand::V2ApiDeleteToolAsset { + bearer, + tool_id, + app_id, + file_name, + res: res_sender, + }) + .await + .map_err(|_| warp::reject::reject())?; + + let result = res_receiver.recv().await.map_err(|_| warp::reject::reject())?; + + match result { + Ok(response) => { + let response = create_success_response(response); + Ok(warp::reply::with_status(warp::reply::json(&response), StatusCode::OK)) + } + Err(error) => Ok(warp::reply::with_status( + warp::reply::json(&error), + StatusCode::from_u16(error.code).unwrap(), + )), + } +} diff --git a/shinkai-libs/shinkai-http-api/src/node_commands.rs b/shinkai-libs/shinkai-http-api/src/node_commands.rs index 48d78e893..d337afa34 100644 --- a/shinkai-libs/shinkai-http-api/src/node_commands.rs +++ b/shinkai-libs/shinkai-http-api/src/node_commands.rs @@ -1074,4 +1074,25 @@ pub enum NodeCommand { state: String, res: Sender>, }, + V2ApiUploadToolAsset { + bearer: String, + tool_id: String, + app_id: String, + file_name: String, + file_data: Vec, + res: Sender>, + }, + V2ApiListToolAssets { + bearer: String, + tool_id: String, + app_id: String, + res: Sender, APIError>>, + }, + V2ApiDeleteToolAsset { + bearer: String, + tool_id: String, + app_id: String, + file_name: String, + res: Sender>, + }, } diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs index 0fcb97824..d5d1a9f90 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs @@ -5,9 +5,9 @@ use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, fs, io, thread}; -use super::tool_output_arg::ToolOutputArg; use super::parameters::Parameters; use super::tool_config::{OAuth, ToolConfig}; +use super::tool_output_arg::ToolOutputArg; use super::tool_playground::{SqlQuery, SqlTable}; use crate::tools::error::ToolError; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -239,6 +239,26 @@ impl DenoTool { support_files.iter().for_each(|(file_name, file_code)| { code_files.insert(format!("{}.ts", file_name), file_code.clone()); }); + // Read all files in the assets directory + let assets_path = Path::new(&node_storage_path) + .join(".tools_storage") + .join(&app_id) + .join("assets"); + + let mut assets_files = Vec::new(); + if assets_path.exists() { + for entry in std::fs::read_dir(assets_path) + .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? + { + let entry = entry.map_err(|e| { + ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)) + })?; + let path = entry.path(); + if path.is_file() { + assets_files.push(path); + } + } + } // Setup the engine with the code files and config let tool = DenoRunner::new( @@ -253,7 +273,7 @@ impl DenoTool { execution_id: tool_id.clone(), code_id: "".to_string(), storage: full_path.clone(), - assets_files: vec![], + assets_files, mount_files: vec![], }, deno_binary_path: PathBuf::from( diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs index 45fae28f8..2ccd484d3 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs @@ -15,10 +15,10 @@ use shinkai_tools_runner::tools::shinkai_node_location::ShinkaiNodeLocation; use shinkai_vector_resources::embeddings::Embedding; use tokio::runtime::Runtime; -use super::tool_output_arg::ToolOutputArg; use super::deno_tools::ToolResult; use super::parameters::Parameters; use super::tool_config::{OAuth, ToolConfig}; +use super::tool_output_arg::ToolOutputArg; use super::tool_playground::{SqlQuery, SqlTable}; #[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] @@ -239,6 +239,26 @@ impl PythonTool { code_files.insert(format!("{}.py", file_name), file_code.clone()); }); + let assets_path = Path::new(&node_storage_path) + .join(".tools_storage") + .join(&app_id) + .join("assets"); + + let mut assets_files = Vec::new(); + if assets_path.exists() { + for entry in std::fs::read_dir(assets_path) + .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? + { + let entry = entry.map_err(|e| { + ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)) + })?; + let path = entry.path(); + if path.is_file() { + assets_files.push(path); + } + } + } + // Setup the engine with the code files and config let tool = PythonRunner::new( CodeFiles { @@ -252,7 +272,7 @@ impl PythonTool { execution_id: tool_id.clone(), code_id: "".to_string(), storage: full_path.clone(), - assets_files: vec![], + assets_files, mount_files: vec![], }, uv_binary_path: PathBuf::from( From 13262fb12b24126420f94a072221e5e294089b18 Mon Sep 17 00:00:00 2001 From: Eddie Date: Fri, 13 Dec 2024 09:55:13 -0300 Subject: [PATCH 2/7] Added Assets to Tools and Copy on Store --- .../src/network/handle_commands_list.rs | 12 +- .../network/v2_api/api_v2_commands_tools.rs | 137 ++++++++++++------ .../tool_execution/execution_deno_dynamic.rs | 4 +- .../execution_python_dynamic.rs | 1 + .../src/api_v2/api_v2_handlers_tools.rs | 6 + .../shinkai-http-api/src/node_commands.rs | 2 + .../src/shinkai_tool_manager.rs | 28 +++- .../shinkai-sqlite/src/tool_playground.rs | 13 +- .../src/tools/deno_tools.rs | 1 + .../src/tools/js_toolkit.rs | 11 +- .../src/tools/python_tools.rs | 1 + .../src/tools/shinkai_tool.rs | 2 + .../src/tools/tool_playground.rs | 1 + 13 files changed, 159 insertions(+), 60 deletions(-) diff --git a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs index 1e0591b9f..55cecdca5 100644 --- a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs +++ b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs @@ -2690,10 +2690,18 @@ impl Node { let _ = Node::v2_api_search_shinkai_tool(db_clone, bearer, query, res).await; }); } - NodeCommand::V2ApiSetPlaygroundTool { bearer, payload, res } => { + NodeCommand::V2ApiSetPlaygroundTool { + bearer, + payload, + tool_id, + app_id, + res, + } => { let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); tokio::spawn(async move { - let _ = Node::v2_api_set_playground_tool(db_clone, bearer, payload, res).await; + let _ = Node::v2_api_set_playground_tool(db_clone, bearer, payload, node_env, tool_id, app_id, res) + .await; }); } NodeCommand::V2ApiListPlaygroundTools { bearer, res } => { diff --git a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs index b4880ffa0..40509939d 100644 --- a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs +++ b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs @@ -384,6 +384,9 @@ impl Node { db: Arc>, bearer: String, payload: ToolPlayground, + node_env: NodeEnvironment, + _tool_id: String, + app_id: String, res: Sender>, ) -> Result<(), NodeError> { // Validate the bearer token @@ -391,34 +394,99 @@ impl Node { return Ok(()); } + let toolkit_name = { + let name = format!( + "{}_{}", + payload + .metadata + .name + .to_lowercase() + .replace(" ", "_") + .replace("-", "_") + .replace(":", "_"), + payload + .metadata + .author + .to_lowercase() + .replace(" ", "_") + .replace("-", "_") + .replace(":", "_") + ); + // Use a regex to filter out unwanted characters + let re = regex::Regex::new(r"[^a-z0-9_]").unwrap(); + re.replace_all(&name, "").to_string() + }; + + let storage_path = node_env.node_storage_path.unwrap_or_default(); + // Check all asset files exist in the {storage}/tool_storage/assets/{app_id}/ + let mut origin_path = PathBuf::from(storage_path.clone()); + origin_path.push("tools_storage"); + origin_path.push(app_id); + origin_path.push("assets"); + if let Some(assets) = payload.assets.clone() { + for file_name in assets { + let mut asset_path: PathBuf = origin_path.clone(); + asset_path.push(file_name.clone()); + if !asset_path.exists() { + let api_error = APIError { + code: StatusCode::BAD_REQUEST.as_u16(), + error: "Bad Request".to_string(), + message: format!("Asset file {} does not exist", file_name.clone()), + }; + let _ = res.send(Err(api_error)).await; + return Ok(()); + } + } + } + + // Copy asset to permanent tool_storage folder {storage}/tool_storage/{toolkit_name}.assets/ + let mut perm_file_path = PathBuf::from(storage_path.clone()); + perm_file_path.push(".tools_storage"); + perm_file_path.push(format!(".{}.assets", toolkit_name)); + if let Err(err) = std::fs::create_dir_all(&perm_file_path) { + let api_error = APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Internal Server Error".to_string(), + message: format!("Failed to create permanent storage directory: {}", err), + }; + let _ = res.send(Err(api_error)).await; + return Ok(()); + } + if let Some(assets) = payload.assets.clone() { + for file_name in assets { + let mut tool_path = origin_path.clone(); + tool_path.push(file_name.clone()); + let mut perm_path = perm_file_path.clone(); + perm_path.push(file_name.clone()); + println!( + "copying {} to {}", + tool_path.to_string_lossy(), + perm_path.to_string_lossy() + ); + let copy_res = std::fs::copy(tool_path, perm_path); + if copy_res.is_err() { + let api_error = APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Internal Server Error".to_string(), + message: format!( + "Failed to copy asset file {} to permanent storage: {}", + file_name.clone(), + copy_res.err().unwrap() + ), + }; + let _ = res.send(Err(api_error)).await; + return Ok(()); + } + } + } + // TODO: check that job_id exists let mut updated_payload = payload.clone(); let shinkai_tool = match payload.language { CodeLanguage::Typescript => { let tool = DenoTool { - toolkit_name: { - let name = format!( - "{}_{}", - payload - .metadata - .name - .to_lowercase() - .replace(" ", "_") - .replace("-", "_") - .replace(":", "_"), - payload - .metadata - .author - .to_lowercase() - .replace(" ", "_") - .replace("-", "_") - .replace(":", "_") - ); - // Use a regex to filter out unwanted characters - let re = regex::Regex::new(r"[^a-z0-9_]").unwrap(); - re.replace_all(&name, "").to_string() - }, + toolkit_name, name: payload.metadata.name.clone(), author: payload.metadata.author.clone(), js_code: payload.code.clone(), @@ -435,33 +503,13 @@ impl Node { sql_tables: Some(payload.metadata.sql_tables), sql_queries: Some(payload.metadata.sql_queries), file_inbox: None, + assets: payload.assets, }; ShinkaiTool::Deno(tool, false) } CodeLanguage::Python => { let tool = PythonTool { - toolkit_name: { - let name = format!( - "{}_{}", - payload - .metadata - .name - .to_lowercase() - .replace(" ", "_") - .replace("-", "_") - .replace(":", "_"), - payload - .metadata - .author - .to_lowercase() - .replace(" ", "_") - .replace("-", "_") - .replace(":", "_") - ); - // Use a regex to filter out unwanted characters - let re = regex::Regex::new(r"[^a-z0-9_]").unwrap(); - re.replace_all(&name, "").to_string() - }, + toolkit_name, name: payload.metadata.name.clone(), author: payload.metadata.author.clone(), py_code: payload.code.clone(), @@ -478,6 +526,7 @@ impl Node { sql_tables: Some(payload.metadata.sql_tables), sql_queries: Some(payload.metadata.sql_queries), file_inbox: None, + assets: payload.assets, }; ShinkaiTool::Python(tool, false) } diff --git a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs index 6e0b6e163..48794ff06 100644 --- a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs +++ b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs @@ -2,11 +2,11 @@ use std::collections::HashMap; use serde_json::{Map, Value}; use shinkai_message_primitives::schemas::shinkai_name::ShinkaiName; -use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use shinkai_tools_primitives::tools::deno_tools::{DenoTool, ToolResult}; use shinkai_tools_primitives::tools::error::ToolError; use shinkai_tools_primitives::tools::parameters::Parameters; use shinkai_tools_primitives::tools::tool_config::{OAuth, ToolConfig}; +use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use super::execution_coordinator::handle_oauth; use crate::utils::environment::fetch_node_environment; @@ -46,6 +46,7 @@ pub async fn execute_deno_tool( sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, }; let mut envs = HashMap::new(); @@ -114,6 +115,7 @@ pub fn check_deno_tool( sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, }; let node_env = fetch_node_environment(); diff --git a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs index 3e2302ea5..b5b45e6a5 100644 --- a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs +++ b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs @@ -48,6 +48,7 @@ pub async fn execute_python_tool( sql_queries: None, file_inbox: None, oauth: oauth.clone(), + assets: None, }; let mut envs = HashMap::new(); diff --git a/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs b/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs index e6fcce541..bc348ec4d 100644 --- a/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs +++ b/shinkai-libs/shinkai-http-api/src/api_v2/api_v2_handlers_tools.rs @@ -87,6 +87,8 @@ pub fn tool_routes( .and(warp::post()) .and(with_sender(node_commands_sender.clone())) .and(warp::header::("authorization")) + .and(warp::header::("x-shinkai-tool-id")) + .and(warp::header::("x-shinkai-app-id")) .and(warp::body::json()) .and_then(set_playground_tool_handler); @@ -713,6 +715,8 @@ pub async fn add_shinkai_tool_handler( pub async fn set_playground_tool_handler( sender: Sender, authorization: String, + tool_id: String, + app_id: String, payload: ToolPlayground, ) -> Result { let bearer = authorization.strip_prefix("Bearer ").unwrap_or("").to_string(); @@ -721,6 +725,8 @@ pub async fn set_playground_tool_handler( .send(NodeCommand::V2ApiSetPlaygroundTool { bearer, payload, + tool_id, + app_id, res: res_sender, }) .await diff --git a/shinkai-libs/shinkai-http-api/src/node_commands.rs b/shinkai-libs/shinkai-http-api/src/node_commands.rs index d337afa34..5c3c93573 100644 --- a/shinkai-libs/shinkai-http-api/src/node_commands.rs +++ b/shinkai-libs/shinkai-http-api/src/node_commands.rs @@ -989,6 +989,8 @@ pub enum NodeCommand { V2ApiSetPlaygroundTool { bearer: String, payload: ToolPlayground, + tool_id: String, + app_id: String, res: Sender>, }, V2ApiListPlaygroundTools { diff --git a/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs b/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs index 4298af208..10963634d 100644 --- a/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs +++ b/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs @@ -614,12 +614,7 @@ impl SqliteManager { is_enabled = ?2, is_network = ?3 WHERE tool_key = ?4", - params![ - cast_slice(&embedding), - is_enabled, - is_network, - tool_key - ], + params![cast_slice(&embedding), is_enabled, is_network, tool_key], )?; Ok(()) @@ -635,11 +630,11 @@ mod tests { use shinkai_message_primitives::schemas::shinkai_tool_offering::UsageType; use shinkai_message_primitives::schemas::wallet_mixed::Asset; use shinkai_message_primitives::schemas::wallet_mixed::NetworkIdentifier; - use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use shinkai_tools_primitives::tools::deno_tools::DenoTool; use shinkai_tools_primitives::tools::deno_tools::ToolResult; use shinkai_tools_primitives::tools::network_tool::NetworkTool; use shinkai_tools_primitives::tools::parameters::Parameters; + use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use shinkai_vector_resources::embeddings::Embedding; use shinkai_vector_resources::model_type::{EmbeddingModelType, OllamaTextEmbeddingsInference}; use std::path::PathBuf; @@ -678,6 +673,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; // Wrap the DenoTool in a ShinkaiTool::Deno variant @@ -741,6 +737,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; let deno_tool_2 = DenoTool { @@ -761,6 +758,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; let deno_tool_3 = DenoTool { @@ -781,6 +779,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; let shinkai_tool_1 = ShinkaiTool::Deno(deno_tool_1, true); @@ -838,6 +837,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; let deno_tool_2 = DenoTool { @@ -858,6 +858,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; let deno_tool_3 = DenoTool { @@ -878,6 +879,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; // Wrap the DenoTools in ShinkaiTool::Deno variants @@ -960,6 +962,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, }; // Wrap the DenoTool in a ShinkaiTool::Deno variant @@ -1004,6 +1007,7 @@ mod tests { sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, }, DenoTool { toolkit_name: "Deno Toolkit".to_string(), @@ -1023,6 +1027,7 @@ mod tests { sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, }, DenoTool { toolkit_name: "Deno Toolkit".to_string(), @@ -1042,6 +1047,7 @@ mod tests { sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, }, ]; @@ -1103,6 +1109,7 @@ mod tests { sql_tables: Some(vec![]), sql_queries: Some(vec![]), file_inbox: None, + assets: None, oauth: None, }; @@ -1124,6 +1131,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; // Add both tools to the database @@ -1171,7 +1179,9 @@ mod tests { deno_tool.activated = false; let updated_tool = ShinkaiTool::Deno(deno_tool, false); // Just update the tool status - no need to regenerate the vector - manager.update_tool_with_vector(updated_tool, SqliteManager::generate_vector_for_testing(0.1)).unwrap(); + manager + .update_tool_with_vector(updated_tool, SqliteManager::generate_vector_for_testing(0.1)) + .unwrap(); } // Search again excluding disabled tools - should now return empty results @@ -1209,6 +1219,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; let disabled_non_network_tool = DenoTool { @@ -1229,6 +1240,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; let usage_type = UsageType::PerUse(ToolPrice::Payment(vec![AssetPayment { diff --git a/shinkai-libs/shinkai-sqlite/src/tool_playground.rs b/shinkai-libs/shinkai-sqlite/src/tool_playground.rs index 197728e29..e3b54eb33 100644 --- a/shinkai-libs/shinkai-sqlite/src/tool_playground.rs +++ b/shinkai-libs/shinkai-sqlite/src/tool_playground.rs @@ -160,6 +160,7 @@ impl SqliteManager { job_id: row.get(8)?, job_id_history: job_id_history.split(',').map(String::from).collect(), code: row.get(10)?, + assets: None, }) }) .map_err(|e| { @@ -223,6 +224,7 @@ impl SqliteManager { job_id: row.get(8)?, job_id_history: job_id_history.split(',').map(String::from).collect(), code: row.get(10)?, + assets: None, }) })?; @@ -258,7 +260,10 @@ impl SqliteManager { mod tests { use super::*; use shinkai_tools_primitives::tools::{ - tool_output_arg::ToolOutputArg, deno_tools::{DenoTool, ToolResult}, parameters::Parameters, shinkai_tool::ShinkaiTool + deno_tools::{DenoTool, ToolResult}, + parameters::Parameters, + shinkai_tool::ShinkaiTool, + tool_output_arg::ToolOutputArg, }; use shinkai_vector_resources::model_type::{EmbeddingModelType, OllamaTextEmbeddingsInference}; use std::path::PathBuf; @@ -293,6 +298,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; let shinkai_tool = ShinkaiTool::Deno(deno_tool, true); @@ -325,6 +331,7 @@ mod tests { job_id: "job_123".to_string(), job_id_history: vec![], code: "console.log('Hello, world!');".to_string(), + assets: None, } } @@ -408,7 +415,6 @@ mod tests { description: "A Deno tool for testing".to_string(), keywords: vec!["deno".to_string(), "test".to_string()], input_args: Parameters::new(), - output_arg: ToolOutputArg::empty(), activated: true, embedding: None, @@ -417,6 +423,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; let shinkai_tool = ShinkaiTool::Deno(deno_tool, true); @@ -480,7 +487,6 @@ mod tests { description: "A Deno tool for testing".to_string(), keywords: vec!["deno".to_string(), "test".to_string()], input_args: Parameters::new(), - output_arg: ToolOutputArg::empty(), activated: true, embedding: None, @@ -489,6 +495,7 @@ mod tests { sql_queries: Some(vec![]), file_inbox: None, oauth: None, + assets: None, }; let shinkai_tool = ShinkaiTool::Deno(deno_tool, true); diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs index d5d1a9f90..5b321a739 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs @@ -41,6 +41,7 @@ pub struct DenoTool { pub sql_queries: Option>, pub file_inbox: Option, pub oauth: Option>, + pub assets: Option>, } impl DenoTool { diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/js_toolkit.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/js_toolkit.rs index 4c90f7414..75d31887f 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/js_toolkit.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/js_toolkit.rs @@ -5,10 +5,10 @@ use shinkai_tools_runner::tools::tool_definition::ToolDefinition; use shinkai_vector_resources::embeddings::Embedding; use super::{ - tool_output_arg::ToolOutputArg, deno_tools::ToolResult, parameters::{Parameters, Property}, tool_config::{BasicConfig, ToolConfig}, + tool_output_arg::ToolOutputArg, }; /// A JSToolkit is a collection of JSTools. @@ -73,6 +73,7 @@ impl JSToolkit { sql_tables: None, sql_queries: None, file_inbox: None, + assets: None, } } @@ -99,7 +100,13 @@ impl JSToolkit { for (key, value) in props { let property_type = value["type"].as_str().unwrap_or("string").to_string(); let description = value["description"].as_str().unwrap_or_default().to_string(); - properties.insert(key.clone(), Property { property_type, description }); + properties.insert( + key.clone(), + Property { + property_type, + description, + }, + ); } } diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs index 2ccd484d3..974995067 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs @@ -40,6 +40,7 @@ pub struct PythonTool { pub sql_queries: Option>, pub file_inbox: Option, pub oauth: Option>, + pub assets: Option>, } impl PythonTool { diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/shinkai_tool.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/shinkai_tool.rs index 33e343849..a60753471 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/shinkai_tool.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/shinkai_tool.rs @@ -419,6 +419,7 @@ mod tests { sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; // Create a ShinkaiTool instance @@ -496,6 +497,7 @@ mod tests { sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; let shinkai_tool = ShinkaiTool::Deno(deno_tool, true); diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/tool_playground.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/tool_playground.rs index b03fa92ca..e1b5ab29c 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/tool_playground.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/tool_playground.rs @@ -28,6 +28,7 @@ pub struct ToolPlayground { pub job_id_history: Vec, pub code: String, pub language: CodeLanguage, + pub assets: Option>, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] From 78fcf667e5c86b529f9ae565640705256936e5b9 Mon Sep 17 00:00:00 2001 From: Eddie Date: Fri, 13 Dec 2024 11:57:37 -0300 Subject: [PATCH 3/7] Fixed paths --- .../network/v2_api/api_v2_commands_tools.rs | 28 +++++++-------- .../tool_execution/execution_deno_dynamic.rs | 23 +++++++++++++ .../execution_python_dynamic.rs | 24 ++++++++++++- .../src/tools/deno_tools.rs | 34 ++++++++----------- .../src/tools/python_tools.rs | 34 ++++++++----------- 5 files changed, 88 insertions(+), 55 deletions(-) diff --git a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs index 40509939d..a6c33b239 100644 --- a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs +++ b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs @@ -419,10 +419,10 @@ impl Node { let storage_path = node_env.node_storage_path.unwrap_or_default(); // Check all asset files exist in the {storage}/tool_storage/assets/{app_id}/ - let mut origin_path = PathBuf::from(storage_path.clone()); - origin_path.push("tools_storage"); + let mut origin_path: PathBuf = PathBuf::from(storage_path.clone()); + origin_path.push(".tools_storage"); + origin_path.push("playground"); origin_path.push(app_id); - origin_path.push("assets"); if let Some(assets) = payload.assets.clone() { for file_name in assets { let mut asset_path: PathBuf = origin_path.clone(); @@ -442,7 +442,8 @@ impl Node { // Copy asset to permanent tool_storage folder {storage}/tool_storage/{toolkit_name}.assets/ let mut perm_file_path = PathBuf::from(storage_path.clone()); perm_file_path.push(".tools_storage"); - perm_file_path.push(format!(".{}.assets", toolkit_name)); + perm_file_path.push("tools"); + perm_file_path.push(toolkit_name.clone()); if let Err(err) = std::fs::create_dir_all(&perm_file_path) { let api_error = APIError { code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), @@ -1806,10 +1807,8 @@ impl Node { let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); file_path.push(".tools_storage"); + file_path.push("playground"); file_path.push(app_id); - // TODO - // keep this in sync with deno/python runner - file_path.push("assets"); // Create directories if they don't exist if !file_path.exists() { std::fs::create_dir_all(&file_path)?; @@ -1842,11 +1841,14 @@ impl Node { let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); file_path.push(".tools_storage"); + file_path.push("playground"); file_path.push(app_id); - // TODO - // keep this in sync with deno/python runner - file_path.push("assets"); - let files = std::fs::read_dir(&file_path).unwrap(); + let files = std::fs::read_dir(&file_path); + if files.is_err() { + let _ = res.send(Ok(vec![])).await; + return Ok(()); + } + let files = files.unwrap(); let file_names = files .map(|file| file.unwrap().file_name().to_string_lossy().to_string()) .collect(); @@ -1870,10 +1872,8 @@ impl Node { let mut file_path = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()); file_path.push(".tools_storage"); + file_path.push("playground"); file_path.push(app_id); - // TODO - // keep this in sync with deno/python runner - file_path.push("assets"); file_path.push(&file_name); let stat = std::fs::remove_file(&file_path).map_err(|err| APIError { code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), diff --git a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs index 48794ff06..b157b5d3e 100644 --- a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs +++ b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_deno_dynamic.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::path::PathBuf; use serde_json::{Map, Value}; use shinkai_message_primitives::schemas::shinkai_name::ShinkaiName; @@ -72,6 +73,27 @@ pub async fn execute_deno_tool( .clone() .ok_or_else(|| ToolError::ExecutionError("Node storage path is not set".to_string()))?; + // Get Assets for Playground; + // Read all files in the assets directory + let assets_path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("playground") + .join(app_id.clone()); + + let mut assets_files = Vec::new(); + if assets_path.exists() { + for entry in std::fs::read_dir(assets_path) + .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? + { + let entry = + entry.map_err(|e| ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)))?; + let path = entry.path(); + if path.is_file() { + assets_files.push(path); + } + } + } + match tool.run_on_demand( envs, node_env.api_listen_address.ip().to_string(), @@ -84,6 +106,7 @@ pub async fn execute_deno_tool( tool_id.clone(), node_name, false, + assets_files, ) { Ok(run_result) => Ok(run_result.data), Err(e) => Err(e), diff --git a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs index b5b45e6a5..ed633ff0d 100644 --- a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs +++ b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_python_dynamic.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use super::execution_coordinator::handle_oauth; use crate::utils::environment::fetch_node_environment; @@ -74,6 +74,27 @@ pub async fn execute_python_tool( .clone() .ok_or_else(|| ToolError::ExecutionError("Node storage path is not set".to_string()))?; + // Get Assets for Playground; + // Read all files in the assets directory + let assets_path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("playground") + .join(app_id.clone()); + + let mut assets_files = Vec::new(); + if assets_path.exists() { + for entry in std::fs::read_dir(assets_path) + .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? + { + let entry = + entry.map_err(|e| ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)))?; + let path = entry.path(); + if path.is_file() { + assets_files.push(path); + } + } + } + match tool.run_on_demand( envs, node_env.api_listen_address.ip().to_string(), @@ -86,6 +107,7 @@ pub async fn execute_python_tool( tool_id.clone(), node_name, false, + assets_files, ) { Ok(run_result) => Ok(run_result.data), Err(e) => Err(e), diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs index 5b321a739..b5d0a35a9 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs @@ -75,6 +75,18 @@ impl DenoTool { node_name: ShinkaiName, is_temporary: bool, ) -> Result { + let path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("tools") + .join(self.toolkit_name.clone()); + let assets_files = self + .assets + .clone() + .unwrap_or(vec![]) + .iter() + .map(|asset| path.clone().join(asset)) + .collect(); + self.run_on_demand( envs, api_ip, @@ -87,6 +99,7 @@ impl DenoTool { tool_id, node_name, is_temporary, + assets_files, ) } @@ -103,6 +116,7 @@ impl DenoTool { tool_id: String, node_name: ShinkaiName, is_temporary: bool, + assets_files: Vec, ) -> Result { println!( "[Running DenoTool] Named: {}, Input: {:?}, Extra Config: {:?}", @@ -240,26 +254,6 @@ impl DenoTool { support_files.iter().for_each(|(file_name, file_code)| { code_files.insert(format!("{}.ts", file_name), file_code.clone()); }); - // Read all files in the assets directory - let assets_path = Path::new(&node_storage_path) - .join(".tools_storage") - .join(&app_id) - .join("assets"); - - let mut assets_files = Vec::new(); - if assets_path.exists() { - for entry in std::fs::read_dir(assets_path) - .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? - { - let entry = entry.map_err(|e| { - ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)) - })?; - let path = entry.path(); - if path.is_file() { - assets_files.push(path); - } - } - } // Setup the engine with the code files and config let tool = DenoRunner::new( diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs index 974995067..1d43131e5 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs @@ -74,6 +74,18 @@ impl PythonTool { node_name: ShinkaiName, is_temporary: bool, ) -> Result { + let path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("tools") + .join(self.toolkit_name.clone()); + let assets_files = self + .assets + .clone() + .unwrap_or(vec![]) + .iter() + .map(|asset| path.clone().join(asset)) + .collect(); + self.run_on_demand( envs, api_ip, @@ -86,6 +98,7 @@ impl PythonTool { tool_id, node_name, is_temporary, + assets_files, ) } @@ -102,6 +115,7 @@ impl PythonTool { tool_id: String, node_name: ShinkaiName, is_temporary: bool, + assets_files: Vec, ) -> Result { println!( "[Running DenoTool] Named: {}, Input: {:?}, Extra Config: {:?}", @@ -240,26 +254,6 @@ impl PythonTool { code_files.insert(format!("{}.py", file_name), file_code.clone()); }); - let assets_path = Path::new(&node_storage_path) - .join(".tools_storage") - .join(&app_id) - .join("assets"); - - let mut assets_files = Vec::new(); - if assets_path.exists() { - for entry in std::fs::read_dir(assets_path) - .map_err(|e| ToolError::ExecutionError(format!("Failed to read assets directory: {}", e)))? - { - let entry = entry.map_err(|e| { - ToolError::ExecutionError(format!("Failed to read directory entry: {}", e)) - })?; - let path = entry.path(); - if path.is_file() { - assets_files.push(path); - } - } - } - // Setup the engine with the code files and config let tool = PythonRunner::new( CodeFiles { From dfaf100124d99b761fcd96bd176ee43523c38755 Mon Sep 17 00:00:00 2001 From: Eddie Date: Fri, 13 Dec 2024 14:05:46 -0300 Subject: [PATCH 4/7] export/import tools --- .../shinkai-node/src/managers/tool_router.rs | 21 +- .../src/network/handle_commands_list.rs | 6 +- .../network/v2_api/api_v2_commands_tools.rs | 215 ++++++++++++------ .../tool_execution/execution_coordinator.rs | 2 + .../src/wallet/coinbase_mpc_wallet.rs | 1 + .../src/tools/deno_tools.rs | 27 ++- .../src/tools/python_tools.rs | 27 ++- 7 files changed, 199 insertions(+), 100 deletions(-) diff --git a/shinkai-bin/shinkai-node/src/managers/tool_router.rs b/shinkai-bin/shinkai-node/src/managers/tool_router.rs index a432884e3..24535a5fa 100644 --- a/shinkai-bin/shinkai-node/src/managers/tool_router.rs +++ b/shinkai-bin/shinkai-node/src/managers/tool_router.rs @@ -23,7 +23,6 @@ use shinkai_message_primitives::shinkai_utils::shinkai_logging::{shinkai_log, Sh use shinkai_sqlite::errors::SqliteManagerError; use shinkai_sqlite::files::prompts_data; use shinkai_sqlite::SqliteManager; -use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use shinkai_tools_primitives::tools::error::ToolError; use shinkai_tools_primitives::tools::js_toolkit::JSToolkit; use shinkai_tools_primitives::tools::network_tool::NetworkTool; @@ -31,6 +30,7 @@ use shinkai_tools_primitives::tools::parameters::Parameters; use shinkai_tools_primitives::tools::rust_tools::RustTool; use shinkai_tools_primitives::tools::shinkai_tool::{ShinkaiTool, ShinkaiToolHeader}; use shinkai_tools_primitives::tools::tool_config::ToolConfig; +use shinkai_tools_primitives::tools::tool_output_arg::ToolOutputArg; use shinkai_tools_runner::built_in_tools; use shinkai_vector_resources::embedding_generator::EmbeddingGenerator; use tokio::sync::RwLock; @@ -216,7 +216,12 @@ impl ToolRouter { config: vec![], input_args: { let mut params = Parameters::new(); - params.add_property("message".to_string(), "string".to_string(), "The message to echo".to_string(), true); + params.add_property( + "message".to_string(), + "string".to_string(), + "The message to echo".to_string(), + true, + ); params }, output_arg: ToolOutputArg { json: "".to_string() }, @@ -446,9 +451,10 @@ impl ToolRouter { function_config_vec, node_storage_path, app_id, - tool_id, + tool_id.clone(), node_name, false, + Some(tool_id), ) .map_err(|e| LLMProviderError::FunctionExecutionError(e.to_string()))?; let result_str = serde_json::to_string(&result) @@ -791,10 +797,11 @@ impl ToolRouter { function_config_vec, node_storage_path, app_id, - tool_id, + tool_id.clone(), // TODO Is this correct? requester_node_name, true, + Some(tool_id), ) .map_err(|e| LLMProviderError::FunctionExecutionError(e.to_string()))?; let result_str = @@ -826,11 +833,7 @@ impl ToolRouter { // Start the timer for FTS search let fts_start_time = Instant::now(); - let fts_search_result = self - .sqlite_manager - .read() - .await - .search_tools_fts(&sanitized_query); + let fts_search_result = self.sqlite_manager.read().await.search_tools_fts(&sanitized_query); let fts_elapsed_time = fts_start_time.elapsed(); println!("Time taken for FTS search: {:?}", fts_elapsed_time); diff --git a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs index 55cecdca5..986f630e4 100644 --- a/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs +++ b/shinkai-bin/shinkai-node/src/network/handle_commands_list.rs @@ -2563,14 +2563,16 @@ impl Node { res, } => { let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); tokio::spawn(async move { - let _ = Node::v2_api_export_tool(db_clone, bearer, tool_key_path, res).await; + let _ = Node::v2_api_export_tool(db_clone, bearer, node_env, tool_key_path, res).await; }); } NodeCommand::V2ApiImportTool { bearer, url, res } => { let db_clone = Arc::clone(&self.db); + let node_env = fetch_node_environment(); tokio::spawn(async move { - let _ = Node::v2_api_import_tool(db_clone, bearer, url, res).await; + let _ = Node::v2_api_import_tool(db_clone, bearer, node_env, url, res).await; }); } NodeCommand::V2ApiResolveShinkaiFileProtocol { diff --git a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs index a6c33b239..13fdb7255 100644 --- a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs +++ b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs @@ -417,6 +417,59 @@ impl Node { re.replace_all(&name, "").to_string() }; + let mut updated_payload = payload.clone(); + + let shinkai_tool = match payload.language { + CodeLanguage::Typescript => { + let tool = DenoTool { + toolkit_name, + name: payload.metadata.name.clone(), + author: payload.metadata.author.clone(), + js_code: payload.code.clone(), + tools: payload.metadata.tools.clone(), + config: payload.metadata.configurations.clone(), + oauth: payload.metadata.oauth.clone(), + description: payload.metadata.description.clone(), + keywords: payload.metadata.keywords.clone(), + input_args: payload.metadata.parameters.clone(), + output_arg: ToolOutputArg { json: "".to_string() }, + activated: false, // TODO: maybe we want to add this as an option in the UI? + embedding: None, + result: payload.metadata.result, + sql_tables: Some(payload.metadata.sql_tables), + sql_queries: Some(payload.metadata.sql_queries), + file_inbox: None, + assets: payload.assets.clone(), + }; + ShinkaiTool::Deno(tool, false) + } + CodeLanguage::Python => { + let tool = PythonTool { + toolkit_name, + name: payload.metadata.name.clone(), + author: payload.metadata.author.clone(), + py_code: payload.code.clone(), + tools: payload.metadata.tools.clone(), + config: payload.metadata.configurations.clone(), + oauth: payload.metadata.oauth.clone(), + description: payload.metadata.description.clone(), + keywords: payload.metadata.keywords.clone(), + input_args: payload.metadata.parameters.clone(), + output_arg: ToolOutputArg { json: "".to_string() }, + activated: false, // TODO: maybe we want to add this as an option in the UI? + embedding: None, + result: payload.metadata.result, + sql_tables: Some(payload.metadata.sql_tables), + sql_queries: Some(payload.metadata.sql_queries), + file_inbox: None, + assets: payload.assets.clone(), + }; + ShinkaiTool::Python(tool, false) + } + }; + + updated_payload.tool_router_key = Some(shinkai_tool.tool_router_key()); + let storage_path = node_env.node_storage_path.unwrap_or_default(); // Check all asset files exist in the {storage}/tool_storage/assets/{app_id}/ let mut origin_path: PathBuf = PathBuf::from(storage_path.clone()); @@ -439,11 +492,11 @@ impl Node { } } - // Copy asset to permanent tool_storage folder {storage}/tool_storage/{toolkit_name}.assets/ + // Copy asset to permanent tool_storage folder {storage}/tool_storage/{tool_key}.assets/ let mut perm_file_path = PathBuf::from(storage_path.clone()); perm_file_path.push(".tools_storage"); perm_file_path.push("tools"); - perm_file_path.push(toolkit_name.clone()); + perm_file_path.push(shinkai_tool.tool_router_key()); if let Err(err) = std::fs::create_dir_all(&perm_file_path) { let api_error = APIError { code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), @@ -481,60 +534,6 @@ impl Node { } } - // TODO: check that job_id exists - let mut updated_payload = payload.clone(); - - let shinkai_tool = match payload.language { - CodeLanguage::Typescript => { - let tool = DenoTool { - toolkit_name, - name: payload.metadata.name.clone(), - author: payload.metadata.author.clone(), - js_code: payload.code.clone(), - tools: payload.metadata.tools.clone(), - config: payload.metadata.configurations.clone(), - oauth: payload.metadata.oauth.clone(), - description: payload.metadata.description.clone(), - keywords: payload.metadata.keywords.clone(), - input_args: payload.metadata.parameters.clone(), - output_arg: ToolOutputArg { json: "".to_string() }, - activated: false, // TODO: maybe we want to add this as an option in the UI? - embedding: None, - result: payload.metadata.result, - sql_tables: Some(payload.metadata.sql_tables), - sql_queries: Some(payload.metadata.sql_queries), - file_inbox: None, - assets: payload.assets, - }; - ShinkaiTool::Deno(tool, false) - } - CodeLanguage::Python => { - let tool = PythonTool { - toolkit_name, - name: payload.metadata.name.clone(), - author: payload.metadata.author.clone(), - py_code: payload.code.clone(), - tools: payload.metadata.tools.clone(), - config: payload.metadata.configurations.clone(), - oauth: payload.metadata.oauth.clone(), - description: payload.metadata.description.clone(), - keywords: payload.metadata.keywords.clone(), - input_args: payload.metadata.parameters.clone(), - output_arg: ToolOutputArg { json: "".to_string() }, - activated: false, // TODO: maybe we want to add this as an option in the UI? - embedding: None, - result: payload.metadata.result, - sql_tables: Some(payload.metadata.sql_tables), - sql_queries: Some(payload.metadata.sql_queries), - file_inbox: None, - assets: payload.assets, - }; - ShinkaiTool::Python(tool, false) - } - }; - - updated_payload.tool_router_key = Some(shinkai_tool.tool_router_key()); - // Function to handle saving metadata and sending response async fn save_metadata_and_respond( db: Arc>, @@ -1525,6 +1524,7 @@ impl Node { pub async fn v2_api_export_tool( db: Arc>, bearer: String, + node_env: NodeEnvironment, tool_key_path: String, res: Sender, APIError>>, ) -> Result<(), NodeError> { @@ -1544,8 +1544,26 @@ impl Node { let mut zip = ZipWriter::new(file); - // TODO Add Assets to the zip file - zip.start_file::<_, ()>("tool.json", FileOptions::default()) + let assets = PathBuf::from(&node_env.node_storage_path.unwrap_or_default()) + .join(".tools_storage") + .join("tools") + .join(tool.tool_router_key()); + if assets.exists() { + for entry in std::fs::read_dir(assets).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_file() { + zip.start_file::<_, ()>( + path.file_name().unwrap().to_str().unwrap(), + FileOptions::default(), + ) + .unwrap(); + zip.write_all(&fs::read(path).await.unwrap()).unwrap(); + } + } + } + + zip.start_file::<_, ()>("__tool.json", FileOptions::default()) .map_err(|e| NodeError::from(e.to_string()))?; zip.write_all(&tool_bytes).map_err(|e| NodeError::from(e.to_string()))?; zip.finish().map_err(|e| NodeError::from(e.to_string()))?; @@ -1573,6 +1591,7 @@ impl Node { pub async fn v2_api_import_tool( db: Arc>, bearer: String, + node_env: NodeEnvironment, url: String, res: Sender>, ) -> Result<(), NodeError> { @@ -1581,7 +1600,7 @@ impl Node { return Ok(()); } - let result = Self::v2_api_import_tool_internal(db, url).await; + let result = Self::v2_api_import_tool_internal(db, node_env, url).await; match result { Ok(response) => { let _ = res.send(Ok(response)).await; @@ -1593,7 +1612,11 @@ impl Node { Ok(()) } - async fn v2_api_import_tool_internal(db: Arc>, url: String) -> Result { + async fn v2_api_import_tool_internal( + db: Arc>, + node_env: NodeEnvironment, + url: String, + ) -> Result { // Download the zip file let response = match reqwest::get(&url).await { Ok(response) => response, @@ -1636,7 +1659,7 @@ impl Node { // Extract and parse tool.json let mut buffer = Vec::new(); { - let mut tool_file = match archive.by_name("tool.json") { + let mut tool_file = match archive.by_name("__tool.json") { Ok(file) => file, Err(_) => { return Err(APIError { @@ -1672,12 +1695,70 @@ impl Node { // Save the tool to the database let mut db_write = db.write().await; match db_write.add_tool(tool).await { - Ok(tool) => Ok(json!({ - "status": "success", - "message": "Tool imported successfully", - "tool_key": tool.tool_router_key(), - "tool": tool - })), + Ok(tool) => { + let archive_clone = archive.clone(); + let files = archive_clone.file_names(); + for file in files { + println!("File: {:?}", file); + if file == "__tool.json" { + continue; + } + let mut buffer = Vec::new(); + { + let file = archive.by_name(file); + let mut tool_file = match file { + Ok(file) => file, + Err(_) => { + return Err(APIError { + code: StatusCode::BAD_REQUEST.as_u16(), + error: "Invalid Tool Archive".to_string(), + message: "Archive does not contain tool.json".to_string(), + }); + } + }; + + // Read the tool file contents into a buffer + if let Err(err) = tool_file.read_to_end(&mut buffer) { + return Err(APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Read Error".to_string(), + message: format!("Failed to read tool.json contents: {}", err), + }); + } + } // `tool_file` goes out of scope here + + let mut file_path = PathBuf::from(&node_env.node_storage_path.clone().unwrap_or_default()) + .join(".tools_storage") + .join("tools") + .join(tool.tool_router_key()); + if !file_path.exists() { + let s = std::fs::create_dir_all(&file_path); + if s.is_err() { + return Err(APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Failed to create directory".to_string(), + message: format!("Failed to create directory: {}", s.err().unwrap()), + }); + } + } + file_path.push(file); + let s = std::fs::write(&file_path, &buffer); + if s.is_err() { + return Err(APIError { + code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + error: "Failed to write file".to_string(), + message: format!("Failed to write file: {}", s.err().unwrap()), + }); + } + } + + Ok(json!({ + "status": "success", + "message": "Tool imported successfully", + "tool_key": tool.tool_router_key(), + "tool": tool + })) + } Err(err) => Err(APIError { code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(), error: "Database Error".to_string(), diff --git a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_coordinator.rs b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_coordinator.rs index f8f8e3c6d..842dcbf17 100644 --- a/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_coordinator.rs +++ b/shinkai-bin/shinkai-node/src/tools/tool_execution/execution_coordinator.rs @@ -209,6 +209,7 @@ pub async fn execute_tool_cmd( tool_id.clone(), node_name, true, + Some(tool_router_key), ) .map(|result| json!(result.data)) } @@ -250,6 +251,7 @@ pub async fn execute_tool_cmd( tool_id.clone(), node_name, true, + Some(tool_router_key), ) .map(|result| json!(result.data)) .map_err(|e| ToolError::ExecutionError(e.to_string())) diff --git a/shinkai-bin/shinkai-node/src/wallet/coinbase_mpc_wallet.rs b/shinkai-bin/shinkai-node/src/wallet/coinbase_mpc_wallet.rs index 7f7fc5203..a87e1237b 100644 --- a/shinkai-bin/shinkai-node/src/wallet/coinbase_mpc_wallet.rs +++ b/shinkai-bin/shinkai-node/src/wallet/coinbase_mpc_wallet.rs @@ -340,6 +340,7 @@ impl CoinbaseMPCWallet { tool_id, node_name, false, + None, ) .map_err(|e| WalletError::FunctionExecutionError(e.to_string()))?; let result_str = diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs index b5d0a35a9..b9f6cd6c6 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/deno_tools.rs @@ -74,18 +74,23 @@ impl DenoTool { tool_id: String, node_name: ShinkaiName, is_temporary: bool, + files_tool_router_key: Option, ) -> Result { - let path = PathBuf::from(&node_storage_path) - .join(".tools_storage") - .join("tools") - .join(self.toolkit_name.clone()); - let assets_files = self - .assets - .clone() - .unwrap_or(vec![]) - .iter() - .map(|asset| path.clone().join(asset)) - .collect(); + let assets_files = match files_tool_router_key { + Some(tool_router_key) => { + let path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("tools") + .join(tool_router_key); + self.assets + .clone() + .unwrap_or(vec![]) + .iter() + .map(|asset| path.clone().join(asset)) + .collect() + } + None => vec![], + }; self.run_on_demand( envs, diff --git a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs index 1d43131e5..ffeef5e44 100644 --- a/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs +++ b/shinkai-libs/shinkai-tools-primitives/src/tools/python_tools.rs @@ -73,18 +73,23 @@ impl PythonTool { tool_id: String, node_name: ShinkaiName, is_temporary: bool, + files_tool_router_key: Option, ) -> Result { - let path = PathBuf::from(&node_storage_path) - .join(".tools_storage") - .join("tools") - .join(self.toolkit_name.clone()); - let assets_files = self - .assets - .clone() - .unwrap_or(vec![]) - .iter() - .map(|asset| path.clone().join(asset)) - .collect(); + let assets_files = match files_tool_router_key { + Some(tool_router_key) => { + let path = PathBuf::from(&node_storage_path) + .join(".tools_storage") + .join("tools") + .join(tool_router_key); + self.assets + .clone() + .unwrap_or(vec![]) + .iter() + .map(|asset| path.clone().join(asset)) + .collect() + } + None => vec![], + }; self.run_on_demand( envs, From 3d5f80983e6a72c53122ae0b63dd5e5625c93c77 Mon Sep 17 00:00:00 2001 From: Eddie Date: Tue, 17 Dec 2024 17:02:05 -0300 Subject: [PATCH 5/7] Added missing args --- shinkai-bin/shinkai-node/src/managers/tool_router.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shinkai-bin/shinkai-node/src/managers/tool_router.rs b/shinkai-bin/shinkai-node/src/managers/tool_router.rs index 5be1a8380..8e354511e 100644 --- a/shinkai-bin/shinkai-node/src/managers/tool_router.rs +++ b/shinkai-bin/shinkai-node/src/managers/tool_router.rs @@ -436,6 +436,7 @@ async def run(c: CONFIG, p: INPUTS) -> OUTPUT: sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; python_tool } @@ -576,6 +577,7 @@ async def run(c: CONFIG, p: INPUTS) -> OUTPUT: tool_id, node_name, false, + None, ) .map_err(|e| LLMProviderError::FunctionExecutionError(e.to_string()))?; let result_str = serde_json::to_string(&result) From 7828dd3712e7af95d43d74337dda444cbf127021 Mon Sep 17 00:00:00 2001 From: Eddie Date: Tue, 17 Dec 2024 19:33:58 -0300 Subject: [PATCH 6/7] Fixed tests --- shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs b/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs index 95b93f7dd..7ed2cb898 100644 --- a/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs +++ b/shinkai-libs/shinkai-sqlite/src/shinkai_tool_manager.rs @@ -1432,6 +1432,7 @@ mod tests { sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; let tool2 = DenoTool { @@ -1452,6 +1453,7 @@ mod tests { sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; let tool3 = DenoTool { @@ -1472,6 +1474,7 @@ mod tests { sql_queries: None, file_inbox: None, oauth: None, + assets: None, }; // Add tools to database with specific vectors From 50ccec569d2ef830a30138511a1b28598c2f873f Mon Sep 17 00:00:00 2001 From: Eddie Date: Wed, 18 Dec 2024 09:06:53 -0300 Subject: [PATCH 7/7] remove rwlock --- .../src/network/v2_api/api_v2_commands_tools.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs index b25e65f78..ee21bf684 100644 --- a/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs +++ b/shinkai-bin/shinkai-node/src/network/v2_api/api_v2_commands_tools.rs @@ -1609,7 +1609,7 @@ impl Node { } async fn v2_api_import_tool_internal( - db: Arc>, + db: Arc, node_env: NodeEnvironment, url: String, ) -> Result { @@ -1864,7 +1864,7 @@ impl Node { } pub async fn v2_api_upload_tool_asset( - db: Arc>, + db: Arc, bearer: String, _tool_id: String, app_id: String, @@ -1901,7 +1901,7 @@ impl Node { } pub async fn v2_api_list_tool_assets( - db: Arc>, + db: Arc, bearer: String, tool_id: String, app_id: String, @@ -1931,7 +1931,7 @@ impl Node { } pub async fn v2_api_delete_tool_asset( - db: Arc>, + db: Arc, bearer: String, tool_id: String, app_id: String,