update

shinkai-bin/shinkai-node/src/llm_provider/parsing_helper.rs

@@ -2,6 +2,7 @@ use super::error::LLMProviderError;
 use super::execution::prompts::general_prompts::JobPromptGenerator;
 use super::job_manager::JobManager;
 use super::llm_stopper::LLMStopper;
+use shinkai_embedding::embedding_generator::EmbeddingGenerator;
 use shinkai_message_primitives::schemas::llm_providers::common_agent_llm_provider::ProviderOrAgent;
 use shinkai_message_primitives::shinkai_utils::shinkai_logging::{shinkai_log, ShinkaiLogLevel, ShinkaiLogOption};
 use shinkai_sqlite::SqliteManager;

shinkai-libs/shinkai-fs/src/file_parser/file_parser.rs

@@ -1,7 +1,8 @@
+use shinkai_embedding::embedding_generator::EmbeddingGenerator;
+
 use super::file_parser_types::TextGroup;
 use super::local_parsing::LocalFileParser;
 
-use crate::embedding_generator::EmbeddingGenerator;
 use crate::shinkai_fs_error::ShinkaiFsError;
 use std::{future::Future, pin::Pin};
 

shinkai-libs/shinkai-fs/src/file_parser/file_parser_grouping.rs

@@ -1,9 +1,9 @@
 use super::file_parser::ShinkaiFileParser;
 use super::file_parser_types::TextGroup;
-use crate::embedding_generator::EmbeddingGenerator;
 use crate::shinkai_fs_error::ShinkaiFsError;
 use keyphrases::KeyPhraseExtractor;
 use regex::Regex;
+use shinkai_embedding::embedding_generator::EmbeddingGenerator;
 use std::collections::HashMap;
 use std::{future::Future, pin::Pin};
 

shinkai-libs/shinkai-fs/src/lib.rs

@@ -2,3 +2,4 @@
 pub mod shinkai_fs_error;
 pub mod shinkai_file_manager;
 pub mod shinkai_file_manager_ops;
+pub mod file_parser;

shinkai-libs/shinkai-fs/src/shinkai_file_manager.rs

@@ -9,6 +9,8 @@ use shinkai_sqlite::SqliteManager;
 use shinkai_message_primitives::schemas::shinkai_fs::ParsedFile;
 
 use crate::shinkai_fs_error::ShinkaiFsError;
+use crate::file_parser::ShinkaiFileParser;
+use crate::embedding_generator::EmbeddingGenerator;
 
 pub struct ShinkaiFileManager;
 
@@ -30,11 +32,12 @@ pub enum FileProcessingMode {
 impl ShinkaiFileManager {
     /// Process file: If not in DB, add it. If supported, generate chunks.
     /// If already processed, consider checking if file changed (not implemented here).
-    pub fn process_file(
+    pub async fn process_file(
         path: ShinkaiPath,
         base_dir: &Path,
         sqlite_manager: &SqliteManager,
         mode: FileProcessingMode,
+        generator: &dyn EmbeddingGenerator,
     ) -> Result<(), ShinkaiFsError> {
         let rel_path = Self::compute_relative_path(&path, base_dir)?;
         let parsed_file = if let Some(pf) = sqlite_manager.get_parsed_file_by_rel_path(&rel_path)? {
@@ -67,9 +70,18 @@ impl ShinkaiFileManager {
         match mode {
             FileProcessingMode::Auto => {
                 // Implement logic for Auto mode
+                let file_buffer = fs::read(path.as_path())?;
+                let text_groups = ShinkaiFileParser::process_file_into_text_groups(
+                    file_buffer,
+                    rel_path.clone(),
+                    1024, // Example max_node_text_size
+                    VRSourceReference::from_file(&rel_path, TextChunkingStrategy::V1)?,
+                ).await?;
+                // Further processing...
             }
             FileProcessingMode::NoParsing => {
-                // Implement logic for NoParsing mode
+                // NoParsing mode: Skip parsing logic
+                // You might still want to update metadata or perform other tasks
             }
             FileProcessingMode::MustParse => {
                 // Implement logic for MustParse mode

shinkai-libs/shinkai-fs/src/shinkai_file_manager_ops.rs

@@ -234,11 +234,9 @@ impl ShinkaiFileManager {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use shinkai_embedding::model_type::{EmbeddingModelType, OllamaTextEmbeddingsInference};
     use std::fs::{self, File};
     use std::io::Read;
-    use std::path::PathBuf;
-    use tempfile::{tempdir, NamedTempFile};
+    use tempfile::tempdir;
 
     #[test]
     fn test_remove_empty_folder() {