From a79af0ee4d8d650c1c0d3d0e54cfdd2d3ca4b271 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Wed, 11 Oct 2023 00:25:23 -0500 Subject: [PATCH 1/2] fix --- scripts/run_node1.sh | 8 +- .../src/shinkai_utils/shinkai_logging.rs | 23 +++- src/db/db_identity.rs | 47 +++++-- src/managers/identity_manager.rs | 121 +++++++++++------- src/network/node_api.rs | 23 +++- src/network/node_api_commands.rs | 21 ++- src/network/node_local_commands.rs | 4 - src/utils/environment.rs | 1 + tests/db_inbox_tests.rs | 1 - 9 files changed, 171 insertions(+), 78 deletions(-) diff --git a/scripts/run_node1.sh b/scripts/run_node1.sh index 63e6db5ac..90e42a838 100755 --- a/scripts/run_node1.sh +++ b/scripts/run_node1.sh @@ -14,12 +14,6 @@ export STARTING_NUM_QR_DEVICES="1" export FIRST_DEVICE_NEEDS_REGISTRATION_CODE="false" # Add these lines to enable all log options -export LOG_BLOCKCHAIN=1 -export LOG_DATABASE=1 -export LOG_IDENTITY=1 -export LOG_API=1 -export LOG_DETAILED_API=1 -export LOG_NODE=1 -export LOG_INTERNAL_API=1 +export LOG_ALL=1 cargo run diff --git a/shinkai-libs/shinkai-message-primitives/src/shinkai_utils/shinkai_logging.rs b/shinkai-libs/shinkai-message-primitives/src/shinkai_utils/shinkai_logging.rs index f5706bda1..6645031dc 100644 --- a/shinkai-libs/shinkai-message-primitives/src/shinkai_utils/shinkai_logging.rs +++ b/shinkai-libs/shinkai-message-primitives/src/shinkai_utils/shinkai_logging.rs @@ -1,11 +1,12 @@ use colored::*; use chrono::Local; -#[derive(PartialEq)] +#[derive(PartialEq, Debug)] pub enum ShinkaiLogOption { Blockchain, Database, Identity, + JobExecution, API, DetailedAPI, Node, @@ -31,6 +32,20 @@ impl ShinkaiLogLevel { } fn active_log_options() -> Vec { + if std::env::var("LOG_ALL").is_ok() { + return vec![ + ShinkaiLogOption::Blockchain, + ShinkaiLogOption::Database, + ShinkaiLogOption::Identity, + ShinkaiLogOption::JobExecution, + ShinkaiLogOption::API, + ShinkaiLogOption::DetailedAPI, + ShinkaiLogOption::Node, + ShinkaiLogOption::InternalAPI, + ShinkaiLogOption::Tests, + ]; + } + let mut active_options = Vec::new(); if std::env::var("LOG_BLOCKCHAIN").is_ok() { active_options.push(ShinkaiLogOption::Blockchain); @@ -56,6 +71,9 @@ fn active_log_options() -> Vec { if std::env::var("LOG_TESTS").is_ok() { active_options.push(ShinkaiLogOption::Tests); } + if std::env::var("LOG_JOB_EXECUTION").is_ok() { + active_options.push(ShinkaiLogOption::JobExecution); + } active_options } @@ -64,7 +82,8 @@ pub fn shinkai_log(option: ShinkaiLogOption, level: ShinkaiLogLevel, message: &s let active_options = active_log_options(); if active_options.contains(&option) { let time = Local::now().format("%Y-%m-%d %H:%M:%S"); - let message_with_time = format!("{} | {}", time, message); + let option_str = format!("{:?}", option); + let message_with_time = format!("{} | {} | {}", time, option_str, message); match level.to_log_level() { log::Level::Error => eprintln!("{}", message_with_time.red()), log::Level::Info => println!("{}", message_with_time.yellow()), diff --git a/src/db/db_identity.rs b/src/db/db_identity.rs index 7c0eb8ef7..336ae8963 100644 --- a/src/db/db_identity.rs +++ b/src/db/db_identity.rs @@ -7,6 +7,7 @@ use shinkai_message_primitives::shinkai_message::shinkai_message_schemas::Identi use shinkai_message_primitives::shinkai_utils::encryption::{ encryption_public_key_to_string, encryption_public_key_to_string_ref, string_to_encryption_public_key, }; +use shinkai_message_primitives::shinkai_utils::shinkai_logging::{shinkai_log, ShinkaiLogOption, ShinkaiLogLevel}; use shinkai_message_primitives::shinkai_utils::signatures::{ signature_public_key_to_string, signature_public_key_to_string_ref, string_to_signature_public_key, }; @@ -256,11 +257,11 @@ impl ShinkaiDB { // Write the batch self.db.write(batch)?; - println!( - "insert_profile::identity.full_identity_name: {}", - identity.full_identity_name + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Info, + format!("Inserted profile: {}", profile_name).as_str(), ); - self.debug_print_all_keys_for_profiles_identity_key(); Ok(()) } @@ -319,7 +320,11 @@ impl ShinkaiDB { let cf_identity = match self.db.cf_handle(Topic::ProfilesIdentityKey.as_str()) { Some(handle) => handle, None => { - eprintln!("Failed to get column family handle for ProfilesIdentityKey"); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!("Failed to get column family handle for ProfilesIdentityKey").as_str(), + ); return; } }; @@ -336,12 +341,20 @@ impl ShinkaiDB { // If it's not, you might get a panic. Consider using // String::from_utf8_lossy if there's a possibility of invalid UTF-8 let key_str = String::from_utf8(key.to_vec()).unwrap(); - println!("print_all_keys_for_profiles_identity_key> {}", key_str); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Debug, + format!("print_all_keys_for_profiles_identity_key {}", key_str).as_str(), + ); self.print_all_devices_for_profile(&key_str); } Err(e) => { // Optionally handle the error, e.g., print it out - eprintln!("Error reading from database: {}", e); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!("Error reading from database: {}", e).as_str(), + ); } } } @@ -352,7 +365,11 @@ impl ShinkaiDB { let cf_device = match self.db.cf_handle(Topic::DevicesIdentityKey.as_str()) { Some(handle) => handle, None => { - eprintln!("Failed to get column family handle for Devices"); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!("Failed to get column family handle for Devices").as_str(), + ); return; } }; @@ -369,12 +386,20 @@ impl ShinkaiDB { // Check if the key (device identity name) contains the profile name if key_str.contains(profile_name) { - println!("print_all_devices_for_profile> {}", key_str); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Debug, + format!("print_all_devices_for_profile {}", key_str).as_str(), + ); } } Err(e) => { // Optionally handle the error, e.g., print it out - eprintln!("Error reading from database: {}", e); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!("Error reading from database: {}", e).as_str(), + ); } } } @@ -478,8 +503,6 @@ impl ShinkaiDB { } pub fn get_profile(&self, full_identity_name: ShinkaiName) -> Result, ShinkaiDBError> { - self.debug_print_all_keys_for_profiles_identity_key(); - let profile_name = full_identity_name .get_profile_name() .ok_or(ShinkaiDBError::InvalidIdentityName(full_identity_name.to_string()))?; diff --git a/src/managers/identity_manager.rs b/src/managers/identity_manager.rs index c7fc1ba39..41306d77a 100644 --- a/src/managers/identity_manager.rs +++ b/src/managers/identity_manager.rs @@ -8,6 +8,7 @@ use shinkai_message_primitives::schemas::agents::serialized_agent::SerializedAge use shinkai_message_primitives::schemas::shinkai_name::ShinkaiName; use shinkai_message_primitives::shinkai_message::shinkai_message::ShinkaiMessage; use shinkai_message_primitives::shinkai_message::shinkai_message_schemas::IdentityPermissions; +use shinkai_message_primitives::shinkai_utils::shinkai_logging::{shinkai_log, ShinkaiLogOption, ShinkaiLogLevel}; use std::sync::Arc; use tokio::sync::Mutex; @@ -32,7 +33,6 @@ impl IdentityManager { .into_iter() .collect() }; - println!("\n\n ### identities_manager identities: {:?}", identities); let agents = { let db = db.lock().await; @@ -41,12 +41,9 @@ impl IdentityManager { .map(Identity::Agent) .collect::>() }; - println!("\n\n ## identities_manager agents: {:?}", agents); - { let db = db.lock().await; db.debug_print_all_keys_for_profiles_identity_key(); - // println!("identities_manager identities: {:?}", identities); } identities.extend(agents); @@ -68,36 +65,54 @@ impl IdentityManager { } pub async fn add_profile_subidentity(&mut self, identity: StandardIdentity) -> anyhow::Result<()> { - eprintln!("add_profile_subidentity > identity: {}", identity); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Info, + format!( + "add_profile_subidentity > identity: {}", + identity + ).as_str() + ); let previously_had_profile_identity = self.has_profile_identity(); self.local_identities.push(Identity::Standard(identity.clone())); if !previously_had_profile_identity && self.has_profile_identity() { - eprintln!("YAY! first profile added!"); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Debug, + format!( + "YAY! first profile added! identity: {}", + identity + ).as_str() + ); self.is_ready = true; } - - { - let db = self.db.lock().await; - db.debug_print_all_keys_for_profiles_identity_key(); - } Ok(()) } pub async fn add_agent_subidentity(&mut self, agent: SerializedAgent) -> anyhow::Result<()> { - println!("add_agent_subidentity > agent: {:?}", agent); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Info, + format!( + "add_agent_subidentity > agent: {:?}", + agent + ).as_str() + ); self.local_identities.push(Identity::Agent(agent.clone())); Ok(()) } pub async fn add_device_subidentity(&mut self, device: DeviceIdentity) -> anyhow::Result<()> { - println!("add_device_subidentity > device: {}", device); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Info, + format!( + "add_device_subidentity > device: {}", + device + ).as_str() + ); self.local_identities.push(Identity::Device(device.clone())); - - { - let db = self.db.lock().await; - db.debug_print_all_keys_for_profiles_identity_key(); - } Ok(()) } @@ -108,11 +123,6 @@ impl IdentityManager { } pub async fn search_local_identity(&self, full_identity_name: &str) -> Option { - { - let db = self.db.lock().await; - db.debug_print_all_keys_for_profiles_identity_key(); - } - let node_in_question = ShinkaiName::new(full_identity_name.to_string()).ok()?.extract_node(); // If the node name matches local node, search in self.identities if self.local_node_name == node_in_question { @@ -121,10 +131,6 @@ impl IdentityManager { .filter_map(|identity| match identity { Identity::Standard(standard_identity) => { if standard_identity.full_identity_name.to_string() == full_identity_name { - println!( - "FOUND! search_local_identity > standard_identity: {}", - standard_identity - ); Some(Identity::Standard(standard_identity.clone())) } else { None @@ -152,7 +158,6 @@ impl IdentityManager { } pub async fn search_local_agent(&self, agent_id: &str) -> Option { - println!("search_local_agent > agent_id: {}", agent_id); let db = self.db.lock().await; db.get_agent(agent_id).ok().flatten() } @@ -163,11 +168,6 @@ impl IdentityManager { } pub async fn search_identity(&self, full_identity_name: &str) -> Option { - println!("search_identity > full_identity_name: {}", full_identity_name); - { - let db = self.db.lock().await; - db.debug_print_all_keys_for_profiles_identity_key(); - } let identity_name = ShinkaiName::new(full_identity_name.to_string()).ok()?; let node_name = identity_name.extract_node(); @@ -218,16 +218,24 @@ impl IdentityManager { } pub async fn external_profile_to_global_identity(&self, full_profile_name: &str) -> Option { - println!( - "external_profile_to_global_identity > full_profile_name: {}", - full_profile_name + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Debug, + format!( + "external_profile_to_global_identity > full_profile_name: {}", + full_profile_name + ).as_str() ); let full_identity_name = match ShinkaiName::new(full_profile_name.to_string().clone()) { Ok(name) => name, Err(_) => { - println!( - "external_profile_to_global_identity > is_valid_node_identity_name_and_no_subidentities: false" + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!( + "external_profile_to_global_identity > is_valid_node_identity_name_and_no_subidentities: false" + ).as_str() ); return None; } @@ -270,9 +278,13 @@ impl IdentityManager { ) -> Result<(), NodeError> { // eprintln!("signature check > sender_subidentity: {:?}", sender_subidentity); if sender_subidentity.is_none() { - eprintln!( - "signature check > Subidentity not found for profile name: {}", - decrypted_message.external_metadata.clone().sender + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!( + "signature check > Subidentity not found for profile name: {}", + decrypted_message.external_metadata.clone().sender + ).as_str() ); return Err(NodeError { message: format!( @@ -290,15 +302,25 @@ impl IdentityManager { Identity::Standard(std_identity) => std_identity.profile_signature_public_key.clone(), Identity::Device(std_device) => Some(std_device.device_signature_public_key.clone()), Identity::Agent(_) => { - eprintln!("signature check > Agent identities cannot send onionized messages"); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!( + "signature check > Agent identities cannot send onionized messages" + ).as_str() + ); return Ok(()); } }; if signature_public_key.is_none() { - eprintln!( - "signature check > Signature public key doesn't exist for identity: {}", - subidentity.get_full_identity_name() + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!( + "signature check > Signature public key doesn't exist for identity: {}", + subidentity.get_full_identity_name() + ).as_str() ); return Err(NodeError { message: format!("Failed to verify message signature. Signature public key doesn't exist for identity"), @@ -308,7 +330,14 @@ impl IdentityManager { match verify_message_signature(signature_public_key.unwrap(), &original_message.clone()) { Ok(_) => Ok({}), Err(e) => { - eprintln!("signature check > Failed to verify message signature: {}", e); + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Error, + format!( + "signature check > Failed to verify message signature: {}", + e.to_string() + ).as_str() + ); return Err(NodeError { message: format!("Failed to verify message signature: {}", e.to_string()), }); diff --git a/src/network/node_api.rs b/src/network/node_api.rs index b5e66dc84..1a964e7dd 100644 --- a/src/network/node_api.rs +++ b/src/network/node_api.rs @@ -1,5 +1,6 @@ use super::node::NodeCommand; use async_channel::Sender; +use chrono::format; use futures::Stream; use futures::StreamExt; use futures::TryFutureExt; @@ -299,7 +300,8 @@ pub async fn run_api(node_commands_sender: Sender, address: SocketA let node_commands_sender = node_commands_sender.clone(); warp::path!("v1" / "add_file_to_inbox_with_symmetric_key" / String / String) .and(warp::post()) - .and(warp::multipart::form()) + .and(warp::body::content_length_limit(1024 * 1024 * 200)) // 200MB + .and(warp::multipart::form().max_length(1024 * 1024 * 200)) .and_then( move |string1: String, string2: String, form: warp::multipart::FormData| { handle_file_upload(node_commands_sender.clone(), string1, string2, form) @@ -448,6 +450,11 @@ async fn handle_file_upload( ) -> Result, warp::Rejection> { let mut stream = Box::pin(form.filter_map(|part_result| async move { if let Ok(part) = part_result { + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Debug, + format!("Received file: {:?}", part).as_str(), + ); if let Some(filename) = part.filename() { let filename = filename.to_string(); let stream = part @@ -802,6 +809,20 @@ async fn handle_rejection(err: warp::Rejection) -> Result() { + let json = warp::reply::json(&APIError::new( + StatusCode::PAYLOAD_TOO_LARGE, + "Payload Too Large", + "The request payload is too large.", + )); + Ok(warp::reply::with_status(json, StatusCode::PAYLOAD_TOO_LARGE)) + } else if let Some(_) = err.find::() { + let json = warp::reply::json(&APIError::new( + StatusCode::BAD_REQUEST, + "Invalid Query", + "The request query string is invalid.", + )); + Ok(warp::reply::with_status(json, StatusCode::BAD_REQUEST)) } else { // Unexpected error, we don't want to expose anything to the user. let json = warp::reply::json(&APIError::new( diff --git a/src/network/node_api_commands.rs b/src/network/node_api_commands.rs index dbc3d27da..ae5d96d68 100644 --- a/src/network/node_api_commands.rs +++ b/src/network/node_api_commands.rs @@ -44,7 +44,7 @@ use shinkai_message_primitives::{ clone_static_secret_key, decrypt_with_chacha20poly1305, encryption_public_key_to_string, encryption_secret_key_to_string, string_to_encryption_public_key, EncryptionMethod, }, - signatures::{clone_signature_secret_key, signature_public_key_to_string, string_to_signature_public_key}, + signatures::{clone_signature_secret_key, signature_public_key_to_string, string_to_signature_public_key}, shinkai_logging::{shinkai_log, ShinkaiLogOption, ShinkaiLogLevel}, }, }; use std::pin::Pin; @@ -127,7 +127,6 @@ impl Node { }) } }; - eprintln!("sender_encryption_pk: {:?}", sender_encryption_pk); msg = match potentially_encrypted_msg .clone() .decrypt_outer_layer(&self.encryption_secret_key, &sender_encryption_pk) @@ -142,12 +141,17 @@ impl Node { } }; } - println!("after decrypt_message_body_if_needed> msg: {:?}", msg); } else { msg = potentially_encrypted_msg.clone(); } } + shinkai_log( + ShinkaiLogOption::Identity, + ShinkaiLogLevel::Info, + format!("after decrypt_message_body_if_needed: {:?}", msg).as_str(), + ); + // Check that the message has the right schema type if let Some(schema) = schema_type { if let Err(e) = msg.validate_message_schema(schema) { @@ -762,7 +766,6 @@ impl Node { } } - db.debug_print_all_keys_for_profiles_identity_key(); let result = db .use_registration_code( &code.clone(), @@ -777,7 +780,6 @@ impl Node { .map(|_| "true".to_string()); println!("handle_registration_code_usage> after use_registration_code"); - db.debug_print_all_keys_for_profiles_identity_key(); std::mem::drop(db); match result { @@ -1636,6 +1638,15 @@ impl Node { } }; + shinkai_log( + ShinkaiLogOption::DetailedAPI, + ShinkaiLogLevel::Debug, + format!( + "api_add_file_to_inbox_with_symmetric_key> filename: {}, hex_blake3_hash: {}, decrypted_file.len(): {}", + filename, hex_blake3_hash, decrypted_file.len() + ).as_str() + ); + match self .db .lock() diff --git a/src/network/node_local_commands.rs b/src/network/node_local_commands.rs index 1a99153d6..a751d7ab4 100644 --- a/src/network/node_local_commands.rs +++ b/src/network/node_local_commands.rs @@ -93,10 +93,6 @@ impl Node { res: Sender, ) -> Result<(), Box> { let db = self.db.lock().await; - - // TODO: remove this - db.debug_print_all_keys_for_profiles_identity_key(); - let code = db .generate_registration_new_code(permissions, code_type) .unwrap_or_else(|_| "".to_string()); diff --git a/src/utils/environment.rs b/src/utils/environment.rs index 5eb9c79de..a86308f9d 100644 --- a/src/utils/environment.rs +++ b/src/utils/environment.rs @@ -1,6 +1,7 @@ use std::env; use std::net::{IpAddr, SocketAddr}; +#[derive(Debug, Clone)] pub struct NodeEnvironment { pub global_identity_name: String, pub listen_address: SocketAddr, diff --git a/tests/db_inbox_tests.rs b/tests/db_inbox_tests.rs index 25746772a..95e03e686 100644 --- a/tests/db_inbox_tests.rs +++ b/tests/db_inbox_tests.rs @@ -306,7 +306,6 @@ fn db_inbox() { let _ = shinkai_db.insert_profile(device1_subidentity.clone()); println!("Inserted profile"); - shinkai_db.debug_print_all_keys_for_profiles_identity_key(); shinkai_db .add_permission(&inbox_name_value, &device1_subidentity, InboxPermission::Admin) From 0db8510fbe49bbbef6073beb2094e8db6a94fcc0 Mon Sep 17 00:00:00 2001 From: Nico Arqueros <1622112+nicarq@users.noreply.github.com> Date: Wed, 11 Oct 2023 00:26:52 -0500 Subject: [PATCH 2/2] new conf --- .vscode/launch.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 67658d690..9c3acfb93 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -39,7 +39,22 @@ } }, "args": [], - "cwd": "${workspaceFolder}" + "cwd": "${workspaceFolder}", + "env": { + "NODE_IP": "0.0.0.0", + "NODE_PORT": "9552", + "NODE_API_IP": "0.0.0.0", + "NODE_API_PORT": "9550", + "IDENTITY_SECRET_KEY": "df3f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81119", + "ENCRYPTION_SECRET_KEY": "d83f619804a92fdb4057192dc43dd748ea778adc52bc498ce80524c014b81159", + "PING_INTERVAL_SECS": "0", + "GLOBAL_IDENTITY_NAME": "@@node1.shinkai", + "RUST_LOG": "debug,error,info", + "STARTING_NUM_QR_PROFILES": "1", + "STARTING_NUM_QR_DEVICES": "1", + "FIRST_DEVICE_NEEDS_REGISTRATION_CODE": "false", + "LOG_ALL": "1" + } }, { "type": "lldb",