cleanup: ignore soft deleted datasets

devflowinc · Aug 22, 2024 · 6a18478 · 6a18478
1 parent f2a7a52
commit 6a18478
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 3 deletions.
diff --git a/server/src/bin/word-worker.rs b/server/src/bin/word-worker.rs
@@ -19,7 +19,7 @@ use trieve_server::{
     errors::ServiceError,
     establish_connection, get_env,
     operators::{
-        chunk_operator::get_check_html_from_ids_query,
+        chunk_operator::get_chunk_html_from_ids_query,
         dataset_operator::add_words_to_dataset,
         parse_operator::convert_html_to_text,
         words_operator::{CreateBkTreeMessage, ProcessWordsFromDatasetMessage},
@@ -250,7 +250,7 @@ async fn process_chunks(
     clickhouse_client: clickhouse::Client,
 ) -> Result<(), ServiceError> {
     let mut word_count_map: HashMap<(uuid::Uuid, String), i32> = HashMap::new();
-    if let Some(chunks) = get_check_html_from_ids_query(
+    if let Some(chunks) = get_chunk_html_from_ids_query(
         message
             .chunks_to_process
             .clone()

diff --git a/server/src/operators/chunk_operator.rs b/server/src/operators/chunk_operator.rs
@@ -2364,7 +2364,7 @@ pub async fn get_pg_point_ids_from_qdrant_point_ids(
 }
 
 #[tracing::instrument(skip(pool))]
-pub async fn get_check_html_from_ids_query(
+pub async fn get_chunk_html_from_ids_query(
     chunk_ids: Vec<uuid::Uuid>,
     pool: web::Data<Pool>,
 ) -> Result<Option<Vec<(uuid::Uuid, String)>>, ServiceError> {

diff --git a/server/src/operators/dataset_operator.rs b/server/src/operators/dataset_operator.rs
@@ -123,6 +123,7 @@ pub async fn get_all_dataset_ids(pool: web::Data<Pool>) -> Result<Vec<uuid::Uuid
 
     let datasets = datasets_columns::datasets
         .select(datasets_columns::id)
+        .filter(datasets_columns::deleted.eq(0))
         .load::<uuid::Uuid>(&mut conn)
         .await
         .map_err(|_| ServiceError::NotFound("Could not find dataset".to_string()))?;