From 69d1886410a6721687f16e2cc3d6669ac5108dbe Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 3 Jun 2022 14:39:20 -0700 Subject: [PATCH] [Type removal] Ignore _type field in bulk request Signed-off-by: Suraj Singh --- .../test/bulk/11_with_deprecated_types.yml | 137 ++++++++++++++++++ .../action/bulk/BulkIntegrationIT.java | 12 ++ .../opensearch/action/bulk/BulkRequest.java | 4 +- .../action/bulk/BulkRequestParserTests.java | 2 +- .../bulk/bulk-with-deprecated-types.json | 9 ++ 5 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/bulk/11_with_deprecated_types.yml create mode 100644 server/src/test/resources/org/opensearch/action/bulk/bulk-with-deprecated-types.json diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/11_with_deprecated_types.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/11_with_deprecated_types.yml new file mode 100644 index 0000000000000..14f5ddcf72ff8 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/bulk/11_with_deprecated_types.yml @@ -0,0 +1,137 @@ +--- +"Array of objects": + - do: + bulk: + refresh: true + body: + - index: + _index: test_index + _type: test_type + _id: test_id + - f1: v1 + f2: 42 + - index: + _index: test_index + _type: test_type + _id: test_id2 + - f1: v2 + f2: 47 + + - do: + count: + index: test_index + + - match: {count: 2} + +--- +"Empty _id": + - do: + bulk: + refresh: true + body: + - index: + _index: test + _type: type + _id: '' + - f: 1 + - index: + _index: test + _type: type + _id: id + - f: 2 + - index: + _index: test + _type: type + - f: 3 + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: illegal_argument_exception } + - match: { items.0.index.error.reason: if _id is specified it must not be empty } + - match: { items.1.index.result: created } + - match: { items.2.index.result: created } + + - do: + count: + index: test + + - match: { count: 2 } + +--- +"Empty _id with op_type create": + - skip: + version: " - 7.4.99" + reason: "auto id + op type create only supported since 7.5" + + - do: + bulk: + refresh: true + body: + - index: + _index: test + _type: type + _id: '' + - f: 1 + - index: + _index: test + _type: type + _id: id + - f: 2 + - index: + _index: test + _type: type + - f: 3 + - create: + _index: test + _type: type + - f: 4 + - index: + _index: test + _type: type + op_type: create + - f: 5 + - match: { errors: true } + - match: { items.0.index.status: 400 } + - match: { items.0.index.error.type: illegal_argument_exception } + - match: { items.0.index.error.reason: if _id is specified it must not be empty } + - match: { items.1.index.result: created } + - match: { items.2.index.result: created } + - match: { items.3.create.result: created } + - match: { items.4.create.result: created } + + - do: + count: + index: test + + - match: { count: 4 } + +--- +"empty action": + - skip: + features: headers + + - do: + catch: /Malformed action\/metadata line \[3\], expected FIELD_NAME but found \[END_OBJECT\]/ + headers: + Content-Type: application/json + bulk: + body: | + {"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}} + {"f1": "v1", "f2": 42} + {} + +--- +"List of strings": + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id"}}' + - '{"f1": "v1", "f2": 42}' + - '{"index": {"_index": "test_index", "_type": "test_type", "_id": "test_id2"}}' + - '{"f1": "v2", "f2": 47}' + + - do: + count: + index: test_index + + - match: {count: 2} diff --git a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java index e2a1363f163da..a2f32c4ddf3c4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/bulk/BulkIntegrationIT.java @@ -170,6 +170,18 @@ public void testBulkWithGlobalDefaults() throws Exception { } } + // Todo: This test is added to verify type support in bulk action. This should be removed once all clients + // avoid sending this param. + // https://github.com/opensearch-project/OpenSearch/issues/3484 + public void testBulkWithTypes() throws Exception { + String bulkAction = copyToStringFromClasspath("/org/opensearch/action/bulk/bulk-with-deprecated-types.json"); + { + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(bulkAction.getBytes(StandardCharsets.UTF_8), 0, bulkAction.length(), null, XContentType.JSON); + assertThat(bulkRequest.numberOfActions(), equalTo(5)); + } + } + private void createSamplePipeline(String pipelineId) throws IOException, ExecutionException, InterruptedException { XContentBuilder pipeline = jsonBuilder().startObject() .startArray("processors") diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java index 25b335eae0bf1..162f3295b9f3a 100644 --- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java +++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java @@ -287,7 +287,9 @@ public BulkRequest add( String routing = valueOrDefault(defaultRouting, globalRouting); String pipeline = valueOrDefault(defaultPipeline, globalPipeline); Boolean requireAlias = valueOrDefault(defaultRequireAlias, globalRequireAlias); - new BulkRequestParser(true).parse( + // https://github.com/opensearch-project/OpenSearch/issues/3484 + // Undo error on types which breaks compatibility with some external clients + new BulkRequestParser(false).parse( data, defaultIndex, routing, diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java index 239bb19c5f6ad..ae4fc5fbd1fa4 100644 --- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java +++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java @@ -231,7 +231,7 @@ public void testFailOnExplicitIndex() { assertEquals("explicit index in bulk is not allowed", ex.getMessage()); } - public void testTypesStillParsedForBulkMonitoring() throws IOException { + public void testTypesStillParsedForExternalClients() throws IOException { BytesArray request = new BytesArray("{ \"index\":{ \"_type\": \"quux\", \"_id\": \"bar\" } }\n{}\n"); BulkRequestParser parser = new BulkRequestParser(false); final AtomicBoolean parsed = new AtomicBoolean(); diff --git a/server/src/test/resources/org/opensearch/action/bulk/bulk-with-deprecated-types.json b/server/src/test/resources/org/opensearch/action/bulk/bulk-with-deprecated-types.json new file mode 100644 index 0000000000000..47ac3353e2d75 --- /dev/null +++ b/server/src/test/resources/org/opensearch/action/bulk/bulk-with-deprecated-types.json @@ -0,0 +1,9 @@ +{"index":{"_index":"logstash-2014.03.30", "_type":"logs"}} +{"message":"in24.inetnebr.com--[01/Aug/1995:00:00:01-0400]\"GET/shuttle/missions/sts-68/news/sts-68-mcc-05.txtHTTP/1.0\"2001839","@version":"1","@timestamp":"2014-03-30T12:38:10.048Z","host":["romeo","in24.inetnebr.com"],"monthday":1,"month":8,"year":1995,"time":"00:00:01","tz":"-0400","request":"\"GET/shuttle/missions/sts-68/news/sts-68-mcc-05.txtHTTP/1.0\"","httpresponse":"200","size":1839,"rtime":"1995-08-01T00:00:01.000Z"} +{ "index":{"_index":"test","_type":"type1","_id":"1"} } +{ "field1" : "value1" } +{ "delete" : { "_index" : "test", "_type" : "type1", "_id" : "2" } } +{ "create" : { "_index" : "test", "_type" : "type1", "_id" : "3" } } +{ "field1" : "value3" } +{ "update" : { "_id" : "0", "_type" : "type1", "_index" : "index1" } } +{ "doc" : {"field" : "updated_value"} }