diff --git a/shinkai-bin/shinkai-node/src/llm_provider/parsing_helper.rs b/shinkai-bin/shinkai-node/src/llm_provider/parsing_helper.rs index 268fa4d44..3641065df 100644 --- a/shinkai-bin/shinkai-node/src/llm_provider/parsing_helper.rs +++ b/shinkai-bin/shinkai-node/src/llm_provider/parsing_helper.rs @@ -2,6 +2,7 @@ use super::error::LLMProviderError; use super::execution::prompts::general_prompts::JobPromptGenerator; use super::job_manager::JobManager; use super::llm_stopper::LLMStopper; +use shinkai_embedding::embedding_generator::EmbeddingGenerator; use shinkai_message_primitives::schemas::llm_providers::common_agent_llm_provider::ProviderOrAgent; use shinkai_message_primitives::shinkai_utils::shinkai_logging::{shinkai_log, ShinkaiLogLevel, ShinkaiLogOption}; use shinkai_sqlite::SqliteManager; diff --git a/shinkai-libs/shinkai-fs/src/file_parser/file_parser.rs b/shinkai-libs/shinkai-fs/src/file_parser/file_parser.rs index 0428a0388..9e5e0a7ca 100644 --- a/shinkai-libs/shinkai-fs/src/file_parser/file_parser.rs +++ b/shinkai-libs/shinkai-fs/src/file_parser/file_parser.rs @@ -1,7 +1,8 @@ +use shinkai_embedding::embedding_generator::EmbeddingGenerator; + use super::file_parser_types::TextGroup; use super::local_parsing::LocalFileParser; -use crate::embedding_generator::EmbeddingGenerator; use crate::shinkai_fs_error::ShinkaiFsError; use std::{future::Future, pin::Pin}; diff --git a/shinkai-libs/shinkai-fs/src/file_parser/file_parser_grouping.rs b/shinkai-libs/shinkai-fs/src/file_parser/file_parser_grouping.rs index 9c685bc7c..99db8b984 100644 --- a/shinkai-libs/shinkai-fs/src/file_parser/file_parser_grouping.rs +++ b/shinkai-libs/shinkai-fs/src/file_parser/file_parser_grouping.rs @@ -1,9 +1,9 @@ use super::file_parser::ShinkaiFileParser; use super::file_parser_types::TextGroup; -use crate::embedding_generator::EmbeddingGenerator; use crate::shinkai_fs_error::ShinkaiFsError; use keyphrases::KeyPhraseExtractor; use regex::Regex; +use shinkai_embedding::embedding_generator::EmbeddingGenerator; use std::collections::HashMap; use std::{future::Future, pin::Pin}; diff --git a/shinkai-libs/shinkai-fs/src/lib.rs b/shinkai-libs/shinkai-fs/src/lib.rs index fd1cce43e..10924825f 100644 --- a/shinkai-libs/shinkai-fs/src/lib.rs +++ b/shinkai-libs/shinkai-fs/src/lib.rs @@ -2,3 +2,4 @@ pub mod shinkai_fs_error; pub mod shinkai_file_manager; pub mod shinkai_file_manager_ops; +pub mod file_parser; diff --git a/shinkai-libs/shinkai-fs/src/shinkai_file_manager.rs b/shinkai-libs/shinkai-fs/src/shinkai_file_manager.rs index 99f1ae63c..1c6d464f2 100644 --- a/shinkai-libs/shinkai-fs/src/shinkai_file_manager.rs +++ b/shinkai-libs/shinkai-fs/src/shinkai_file_manager.rs @@ -9,6 +9,8 @@ use shinkai_sqlite::SqliteManager; use shinkai_message_primitives::schemas::shinkai_fs::ParsedFile; use crate::shinkai_fs_error::ShinkaiFsError; +use crate::file_parser::ShinkaiFileParser; +use crate::embedding_generator::EmbeddingGenerator; pub struct ShinkaiFileManager; @@ -30,11 +32,12 @@ pub enum FileProcessingMode { impl ShinkaiFileManager { /// Process file: If not in DB, add it. If supported, generate chunks. /// If already processed, consider checking if file changed (not implemented here). - pub fn process_file( + pub async fn process_file( path: ShinkaiPath, base_dir: &Path, sqlite_manager: &SqliteManager, mode: FileProcessingMode, + generator: &dyn EmbeddingGenerator, ) -> Result<(), ShinkaiFsError> { let rel_path = Self::compute_relative_path(&path, base_dir)?; let parsed_file = if let Some(pf) = sqlite_manager.get_parsed_file_by_rel_path(&rel_path)? { @@ -67,9 +70,18 @@ impl ShinkaiFileManager { match mode { FileProcessingMode::Auto => { // Implement logic for Auto mode + let file_buffer = fs::read(path.as_path())?; + let text_groups = ShinkaiFileParser::process_file_into_text_groups( + file_buffer, + rel_path.clone(), + 1024, // Example max_node_text_size + VRSourceReference::from_file(&rel_path, TextChunkingStrategy::V1)?, + ).await?; + // Further processing... } FileProcessingMode::NoParsing => { - // Implement logic for NoParsing mode + // NoParsing mode: Skip parsing logic + // You might still want to update metadata or perform other tasks } FileProcessingMode::MustParse => { // Implement logic for MustParse mode diff --git a/shinkai-libs/shinkai-fs/src/shinkai_file_manager_ops.rs b/shinkai-libs/shinkai-fs/src/shinkai_file_manager_ops.rs index 4ec4a1419..5d9b037eb 100644 --- a/shinkai-libs/shinkai-fs/src/shinkai_file_manager_ops.rs +++ b/shinkai-libs/shinkai-fs/src/shinkai_file_manager_ops.rs @@ -234,11 +234,9 @@ impl ShinkaiFileManager { #[cfg(test)] mod tests { use super::*; - use shinkai_embedding::model_type::{EmbeddingModelType, OllamaTextEmbeddingsInference}; use std::fs::{self, File}; use std::io::Read; - use std::path::PathBuf; - use tempfile::{tempdir, NamedTempFile}; + use tempfile::tempdir; #[test] fn test_remove_empty_folder() {