Allow empty string when validation

Signed-off-by: Yizhe Liu <[email protected]>
opensearch-project · Dec 24, 2024 · 27b9580 · 27b9580
1 parent 32ac2cc
commit 27b9580
Show file tree

Hide file tree

Showing 3 changed files with 1 addition and 83 deletions.
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/InferenceProcessor.java
@@ -391,7 +391,7 @@ private void validateEmbeddingFieldsValue(IngestDocument ingestDocument) {
             indexName,
             clusterService,
             environment,
-            false
+            true
         );
     }
 

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/InferenceProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/InferenceProcessorTests.java
@@ -66,28 +66,6 @@ public void test_batchExecute_emptyInput() {
         verify(clientAccessor, never()).inferenceSentences(anyString(), anyList(), any());
     }
 
-    public void test_batchExecuteWithEmpty_allFailedValidation() {
-        final int docCount = 2;
-        TestInferenceProcessor processor = new TestInferenceProcessor(createMockVectorResult(), BATCH_SIZE, null);
-        List<IngestDocumentWrapper> wrapperList = createIngestDocumentWrappers(docCount);
-        wrapperList.get(0).getIngestDocument().setFieldValue("key1", Arrays.asList("", "value1"));
-        wrapperList.get(1).getIngestDocument().setFieldValue("key1", Arrays.asList("", "value1"));
-        Consumer resultHandler = mock(Consumer.class);
-        processor.batchExecute(wrapperList, resultHandler);
-        ArgumentCaptor<List<IngestDocumentWrapper>> captor = ArgumentCaptor.forClass(List.class);
-        verify(resultHandler).accept(captor.capture());
-        assertEquals(docCount, captor.getValue().size());
-        for (int i = 0; i < docCount; ++i) {
-            assertNotNull(captor.getValue().get(i).getException());
-            assertEquals(
-                "list type field [key1] has empty string, cannot process it",
-                captor.getValue().get(i).getException().getMessage()
-            );
-            assertEquals(wrapperList.get(i).getIngestDocument(), captor.getValue().get(i).getIngestDocument());
-        }
-        verify(clientAccessor, never()).inferenceSentences(anyString(), anyList(), any());
-    }
-
     public void test_batchExecuteWithNull_allFailedValidation() {
         final int docCount = 2;
         TestInferenceProcessor processor = new TestInferenceProcessor(createMockVectorResult(), BATCH_SIZE, null);
@@ -107,27 +85,6 @@ public void test_batchExecuteWithNull_allFailedValidation() {
         verify(clientAccessor, never()).inferenceSentences(anyString(), anyList(), any());
     }
 
-    public void test_batchExecute_partialFailedValidation() {
-        final int docCount = 2;
-        TestInferenceProcessor processor = new TestInferenceProcessor(createMockVectorResult(), BATCH_SIZE, null);
-        List<IngestDocumentWrapper> wrapperList = createIngestDocumentWrappers(docCount);
-        wrapperList.get(0).getIngestDocument().setFieldValue("key1", Arrays.asList("", "value1"));
-        wrapperList.get(1).getIngestDocument().setFieldValue("key1", Arrays.asList("value3", "value4"));
-        Consumer resultHandler = mock(Consumer.class);
-        processor.batchExecute(wrapperList, resultHandler);
-        ArgumentCaptor<List<IngestDocumentWrapper>> captor = ArgumentCaptor.forClass(List.class);
-        verify(resultHandler).accept(captor.capture());
-        assertEquals(docCount, captor.getValue().size());
-        assertNotNull(captor.getValue().get(0).getException());
-        assertNull(captor.getValue().get(1).getException());
-        for (int i = 0; i < docCount; ++i) {
-            assertEquals(wrapperList.get(i).getIngestDocument(), captor.getValue().get(i).getIngestDocument());
-        }
-        ArgumentCaptor<List<String>> inferenceTextCaptor = ArgumentCaptor.forClass(List.class);
-        verify(clientAccessor).inferenceSentences(anyString(), inferenceTextCaptor.capture(), any());
-        assertEquals(2, inferenceTextCaptor.getValue().size());
-    }
-
     public void test_batchExecute_happyCase() {
         final int docCount = 2;
         List<List<Float>> inferenceResults = createMockVectorWithLength(6);

diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextEmbeddingProcessorTests.java
@@ -241,31 +241,6 @@ public void testExecute_withListTypeInput_successful() {
         verify(handler).accept(any(IngestDocument.class), isNull());
     }
 
-    public void testExecute_SimpleTypeWithEmptyStringValue_throwIllegalArgumentException() {
-        Map<String, Object> sourceAndMetadata = new HashMap<>();
-        sourceAndMetadata.put(IndexFieldMapper.NAME, "my_index");
-        sourceAndMetadata.put("key1", "    ");
-        IngestDocument ingestDocument = new IngestDocument(sourceAndMetadata, new HashMap<>());
-        TextEmbeddingProcessor processor = createInstanceWithLevel1MapConfig();
-
-        BiConsumer handler = mock(BiConsumer.class);
-        processor.execute(ingestDocument, handler);
-        verify(handler).accept(isNull(), any(IllegalArgumentException.class));
-    }
-
-    public void testExecute_listHasEmptyStringValue_throwIllegalArgumentException() {
-        List<String> list1 = ImmutableList.of("", "test2", "test3");
-        Map<String, Object> sourceAndMetadata = new HashMap<>();
-        sourceAndMetadata.put(IndexFieldMapper.NAME, "my_index");
-        sourceAndMetadata.put("key1", list1);
-        IngestDocument ingestDocument = new IngestDocument(sourceAndMetadata, new HashMap<>());
-        TextEmbeddingProcessor processor = createInstanceWithLevel1MapConfig();
-
-        BiConsumer handler = mock(BiConsumer.class);
-        processor.execute(ingestDocument, handler);
-        verify(handler).accept(isNull(), any(IllegalArgumentException.class));
-    }
-
     public void testExecute_listHasNonStringValue_throwIllegalArgumentException() {
         List<Integer> list2 = ImmutableList.of(1, 2, 3);
         Map<String, Object> sourceAndMetadata = new HashMap<>();
@@ -550,20 +525,6 @@ public void testExecute_mapHasNonStringValue_throwIllegalArgumentException() {
         verify(handler).accept(isNull(), any(IllegalArgumentException.class));
     }
 
-    public void testExecute_mapHasEmptyStringValue_throwIllegalArgumentException() {
-        Map<String, String> map1 = ImmutableMap.of("test1", "test2");
-        Map<String, String> map2 = ImmutableMap.of("test3", "   ");
-        Map<String, Object> sourceAndMetadata = new HashMap<>();
-        sourceAndMetadata.put(IndexFieldMapper.NAME, "my_index");
-        sourceAndMetadata.put("key1", map1);
-        sourceAndMetadata.put("key2", map2);
-        IngestDocument ingestDocument = new IngestDocument(sourceAndMetadata, new HashMap<>());
-        TextEmbeddingProcessor processor = createInstanceWithLevel2MapConfig();
-        BiConsumer handler = mock(BiConsumer.class);
-        processor.execute(ingestDocument, handler);
-        verify(handler).accept(isNull(), any(IllegalArgumentException.class));
-    }
-
     public void testExecute_mapDepthReachLimit_throwIllegalArgumentException() {
         Map<String, Object> ret = createMaxDepthLimitExceedMap(() -> 1);
         Map<String, Object> sourceAndMetadata = new HashMap<>();
-Original file line number
+Diff line change
@@ Expand Up @@
                 indexName,
                 clusterService,
                 environment,
-                false
+                true
             );
         }
@@ Expand Down @@