Merge pull request #7 from xinyual/addMaxChunkForDelimiter

change logic of max chunk number
yuye-aws · Mar 1, 2024 · 001d421 · 001d421
2 parents 1aa3cb4 + 165dfa8
commit 001d421
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 6 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java b/src/main/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunker.java
@@ -31,17 +31,17 @@ public void validateParameters(Map<String, Object> parameters) {
         if (parameters.containsKey(MAX_CHUNK_LIMIT_FIELD)) {
             Object maxChunkLimit = parameters.get(MAX_CHUNK_LIMIT_FIELD);
             if (!(maxChunkLimit instanceof Integer)) {
-                throw new IllegalArgumentException(
-                    "Parameter max_chunk_limit:" + maxChunkLimit.toString() + " cannot be converted to integer."
-                );
+                throw new IllegalArgumentException("Parameter max_chunk_limit:" + maxChunkLimit.toString() + " should be integer.");
+            } else if ((int) maxChunkLimit <= 0) {
+                throw new IllegalArgumentException("Parameter max_chunk_limit:" + maxChunkLimit + " is not greater than 0.");
             }
         }
     }
 
     @Override
     public List<String> chunk(String content, Map<String, Object> parameters) {
         String delimiter = (String) parameters.get(DELIMITER_FIELD);
-        int maxChunkingNumber = (int) parameters.getOrDefault(MAX_CHUNK_LIMIT_FIELD, 0);
+        int maxChunkingNumber = (int) parameters.getOrDefault(MAX_CHUNK_LIMIT_FIELD, -1);
         List<String> chunkResult = new ArrayList<>();
         int start = 0;
         int end = content.indexOf(delimiter);

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunkerTests.java b/src/test/java/org/opensearch/neuralsearch/processor/chunker/DelimiterChunkerTests.java
@@ -30,15 +30,23 @@ public void testChunkerWithWrongLimitFieldList() {
         String content = "a\nb\nc\nd";
         Map<String, Object> inputParameters = Map.of(MAX_CHUNK_LIMIT_FIELD, List.of("-1"), DELIMITER_FIELD, "\n");
         Exception exception = assertThrows(IllegalArgumentException.class, () -> chunker.validateParameters(inputParameters));
-        Assert.assertEquals("Parameter max_chunk_limit:" + List.of("-1") + " cannot be converted to integer.", exception.getMessage());
+        Assert.assertEquals("Parameter max_chunk_limit:" + List.of("-1") + " should be integer.", exception.getMessage());
     }
 
     public void testChunkerWithWrongLimitField() {
         DelimiterChunker chunker = new DelimiterChunker();
         String content = "a\nb\nc\nd";
         Map<String, Object> inputParameters = Map.of(MAX_CHUNK_LIMIT_FIELD, "1000\n", DELIMITER_FIELD, "\n");
         Exception exception = assertThrows(IllegalArgumentException.class, () -> chunker.validateParameters(inputParameters));
-        Assert.assertEquals("Parameter max_chunk_limit:1000\n cannot be converted to integer.", exception.getMessage());
+        Assert.assertEquals("Parameter max_chunk_limit:1000\n should be integer.", exception.getMessage());
+    }
+
+    public void testChunkerWithNegativeLimit() {
+        DelimiterChunker chunker = new DelimiterChunker();
+        String content = "a\nb\nc\nd";
+        Map<String, Object> inputParameters = Map.of(MAX_CHUNK_LIMIT_FIELD, -1, DELIMITER_FIELD, "\n");
+        Exception exception = assertThrows(IllegalArgumentException.class, () -> chunker.validateParameters(inputParameters));
+        Assert.assertEquals("Parameter max_chunk_limit:-1 is not greater than 0.", exception.getMessage());
     }
 
     public void testChunkerWithDelimiterFieldNotString() {