bug fix: only update runtime max chunk limit when enabled

Signed-off-by: yuye-aws <[email protected]>
yuye-aws · Mar 18, 2024 · 68fef4f · 68fef4f
1 parent fb6a961
commit 68fef4f
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 2 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java
@@ -277,10 +277,12 @@ private int chunkMapType(
      * Chunk the content, update the runtime max_chunk_limit and return the result
      */
     private List<String> chunkString(final String content, final Map<String, Object> runTimeParameters) {
-        // update runtime max_chunk_limit for each content
+        // update runtime max_chunk_limit if not disabled
         List<String> contentResult = chunker.chunk(content, runTimeParameters);
         int runtimeMaxChunkLimit = parseIntegerParameter(runTimeParameters, MAX_CHUNK_LIMIT_FIELD, maxChunkLimit);
-        runTimeParameters.put(MAX_CHUNK_LIMIT_FIELD, runtimeMaxChunkLimit - contentResult.size());
+        if (runtimeMaxChunkLimit != DISABLED_MAX_CHUNK_LIMIT) {
+            runTimeParameters.put(MAX_CHUNK_LIMIT_FIELD, runtimeMaxChunkLimit - contentResult.size());
+        }
         return contentResult;
     }
 

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorTests.java
@@ -455,6 +455,14 @@ public void testExecute_withFixedTokenLength_andSourceDataListExceedMaxChunkLimi
             ));
     }
 
+    @SneakyThrows
+    public void testExecute_withFixedTokenLength_andSourceDataListDisabledMaxChunkLimit_thenFail() {
+        int maxChunkLimit = -1;
+        TextChunkingProcessor processor = createFixedTokenLengthInstanceWithMaxChunkLimit(createStringFieldMap(), maxChunkLimit);
+        IngestDocument ingestDocument = createIngestDocumentWithSourceData(createSourceDataListStrings());
+        processor.execute(ingestDocument);
+    }
+
     @SneakyThrows
     public void testCreate_withDefaultAlgorithm_andSourceDataString_thenSucceed() {
         TextChunkingProcessor processor = createDefaultAlgorithmInstance(createStringFieldMap());