From 3d88c8cd8aa54ae3fc95f7a6d17680435b13094c Mon Sep 17 00:00:00 2001 From: lopagela Date: Sun, 5 Nov 2023 13:51:13 +0100 Subject: [PATCH] PR feedback: loading the IndexStore from FS to delete objects from it --- private_gpt/server/ingest/ingest_router.py | 8 +------- private_gpt/server/ingest/ingest_service.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/private_gpt/server/ingest/ingest_router.py b/private_gpt/server/ingest/ingest_router.py index 4841fa82a..7a5231ab0 100644 --- a/private_gpt/server/ingest/ingest_router.py +++ b/private_gpt/server/ingest/ingest_router.py @@ -58,10 +58,4 @@ def delete_ingested(doc_id: str) -> None: from the directory specified in your configuration) """ service = root_injector.get(IngestService) - try: - service.delete(doc_id) - except ValueError as err: - raise HTTPException( - 404, f"Document={doc_id} not found in the datastore" - ) from err - return + service.delete(doc_id) diff --git a/private_gpt/server/ingest/ingest_service.py b/private_gpt/server/ingest/ingest_service.py index b805cd8e6..bc938eac7 100644 --- a/private_gpt/server/ingest/ingest_service.py +++ b/private_gpt/server/ingest/ingest_service.py @@ -10,6 +10,7 @@ StorageContext, StringIterableReader, VectorStoreIndex, + load_index_from_storage, ) from llama_index.node_parser import SentenceWindowNodeParser from llama_index.readers.file.base import DEFAULT_FILE_READER_CLS @@ -170,9 +171,15 @@ def delete(self, doc_id: str) -> None: :raises ValueError: if the document does not exist """ - logger.info("Deleting the ingested document=%s in the doc store", doc_id) - self.storage_context.docstore.delete_ref_doc(doc_id) - # FIXME the documents are only deleted, and not in the vector store - # or index store - # self.storage_context.vector_store.delete(doc_id) - # self.storage_context.index_store.delete_index_struct(doc_id) + logger.info( + "Deleting the ingested document=%s in the doc and index store", doc_id + ) + + # Load the index with store_nodes_override=True to be able to delete them + index = load_index_from_storage(self.storage_context, store_nodes_override=True) + + # Delete the document from the index + index.delete_ref_doc(doc_id, delete_from_docstore=True) + + # Save the index + self.storage_context.persist(persist_dir=local_data_path)