[Type removal] Ignore _type field in bulk request

Signed-off-by: Suraj Singh <[email protected]>

rest-api-spec/src/main/resources/rest-api-spec/test/bulk/11_with_deprecated_types.yml

@@ -0,0 +1,137 @@
+---
+"Array of objects":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test_index
+              _type:  test_type
+              _id:    test_id
+          - f1: v1
+            f2: 42
+          - index:
+              _index: test_index
+              _type:  test_type
+              _id:    test_id2
+          - f1: v2
+            f2: 47
+
+  - do:
+      count:
+        index: test_index
+
+  - match: {count: 2}
+
+---
+"Empty _id":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test
+              _type: type
+              _id: ''
+          - f: 1
+          - index:
+              _index: test
+              _type: type
+              _id: id
+          - f: 2
+          - index:
+              _index: test
+              _type: type
+          - f: 3
+  - match: { errors: true }
+  - match: { items.0.index.status: 400 }
+  - match: { items.0.index.error.type: illegal_argument_exception }
+  - match: { items.0.index.error.reason: if _id is specified it must not be empty }
+  - match: { items.1.index.result: created }
+  - match: { items.2.index.result: created }
+
+  - do:
+      count:
+        index: test
+
+  - match: { count: 2 }
+
+---
+"Empty _id with op_type create":
+  - skip:
+      version: " - 7.4.99"
+      reason: "auto id + op type create only supported since 7.5"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - index:
+              _index: test
+              _type: type
+              _id: ''
+          - f: 1
+          - index:
+              _index: test
+              _type: type
+              _id: id
+          - f: 2
+          - index:
+              _index: test
+              _type: type
+          - f: 3
+          - create:
+              _index: test
+              _type: type
+          - f: 4
+          - index:
+              _index: test
+              _type: type
+              op_type: create
+          - f: 5
+  - match: { errors: true }
+  - match: { items.0.index.status: 400 }
+  - match: { items.0.index.error.type: illegal_argument_exception }
+  - match: { items.0.index.error.reason: if _id is specified it must not be empty }
+  - match: { items.1.index.result: created }
+  - match: { items.2.index.result: created }
+  - match: { items.3.create.result: created }
+  - match: { items.4.create.result: created }
+
+  - do:
+      count:
+        index: test
+
+  - match: { count: 4 }
+
+---
+"empty action":
+  - skip:
+      features: headers
+
+  - do:
+      catch: /Malformed action\/metadata line \[3\], expected FIELD_NAME but found \[END_OBJECT\]/
+      headers:
+        Content-Type: application/json
+      bulk:
+        body: |
+          {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}
+          {"f1": "v1", "f2": 42}
+          {}
+
+---
+"List of strings":
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}'
+          - '{"f1": "v1", "f2": 42}'
+          - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}'
+          - '{"f1": "v2", "f2": 47}'
+
+  - do:
+      count:
+        index: test_index
+
+  - match: {count: 2}

server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java

@@ -170,6 +170,18 @@ public void testBulkWithGlobalDefaults() throws Exception {
         }
     }
 
+    // Todo: This test is added to verify type support in bulk action. This should be removed once all clients
+    // avoid sending this param.
+    // https://github.com/opensearch-project/OpenSearch/issues/3484
+    public void testBulkWithTypes() throws Exception {
+        String bulkAction = copyToStringFromClasspath("/org/opensearch/action/bulk/bulk-with-deprecated-types.json");
+        {
+            BulkRequest bulkRequest = new BulkRequest();
+            bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON);
+            assertThat(bulkRequest.numberOfActions(), equalTo(5));
+        }
+    }
+
     private void createSamplePipeline(String pipelineId) throws IOException, ExecutionException, InterruptedException {
         XContentBuilder pipeline = jsonBuilder().startObject()
             .startArray("processors")

server/src/main/java/org/opensearch/action/bulk/BulkRequest.java

@@ -287,7 +287,9 @@ public BulkRequest add(
         String routing = valueOrDefault(defaultRouting, globalRouting);
         String pipeline = valueOrDefault(defaultPipeline, globalPipeline);
         Boolean requireAlias = valueOrDefault(defaultRequireAlias, globalRequireAlias);
-        new BulkRequestParser(true).parse(
+        // https://github.com/opensearch-project/OpenSearch/issues/3484
+        // Undo error on types which breaks compatibility with some external clients
+        new BulkRequestParser(false).parse(
             data,
             defaultIndex,
             routing,

server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java

@@ -231,7 +231,7 @@ public void testFailOnExplicitIndex() {
         assertEquals("explicit index in bulk is not allowed", ex.getMessage());
     }
 
-    public void testTypesStillParsedForBulkMonitoring() throws IOException {
+    public void testTypesStillParsedForExternalClients() throws IOException {
         BytesArray request = new BytesArray("{ \"index\":{ \"_type\": \"quux\", \"_id\": \"bar\" } }\n{}\n");
         BulkRequestParser parser = new BulkRequestParser(false);
         final AtomicBoolean parsed = new AtomicBoolean();

server/src/test/resources/org/opensearch/action/bulk/bulk-with-deprecated-types.json

@@ -0,0 +1,9 @@
+{"index":{"_index":"logstash-2014.03.30", "_type":"logs"}}
+{"message":"in24.inetnebr.com--[01/Aug/1995:00:00:01-0400]\"GET/shuttle/missions/sts-68/news/sts-68-mcc-05.txtHTTP/1.0\"2001839","@version":"1","@timestamp":"2014-03-30T12:38:10.048Z","host":["romeo","in24.inetnebr.com"],"monthday":1,"month":8,"year":1995,"time":"00:00:01","tz":"-0400","request":"\"GET/shuttle/missions/sts-68/news/sts-68-mcc-05.txtHTTP/1.0\"","httpresponse":"200","size":1839,"rtime":"1995-08-01T00:00:01.000Z"}
+{ "index":{"_index":"test","_type":"type1","_id":"1"} }
+{ "field1" : "value1" }
+{ "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } }
+{ "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } }
+{ "field1" : "value3" }
+{ "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1" } }
+{ "doc" : {"field" : "updated_value"} }