fix: Ignore some emtpy page_content when append to split_documents (#…

…2898)
langgenius · Mar 19, 2024 · 696efe4 · 696efe4
1 parent 4419d35
commit 696efe4
Showing 1 changed file with 4 additions and 3 deletions.
diff --git a/api/core/rag/index_processor/processor/paragraph_index_processor.py b/api/core/rag/index_processor/processor/paragraph_index_processor.py
@@ -45,11 +45,12 @@ def transform(self, documents: list[Document], **kwargs) -> list[Document]:
  # delete Spliter character
  page_content = document_node.page_content
  if page_content.startswith(".") or page_content.startswith("。"):
- page_content = page_content[1:]
+ page_content = page_content[1:].strip()
  else:
  page_content = page_content
- document_node.page_content = page_content
- split_documents.append(document_node)
+ if len(page_content) > 0:
+ document_node.page_content = page_content
+ split_documents.append(document_node)
  all_documents.extend(split_documents)
  return all_documents