Added support for application/x-ndjson and tests for _bulk.

Signed-off-by: dblock <[email protected]>

.cspell

@@ -20,6 +20,7 @@ Lovins
 Lucene
 Millis
 Multisearch
+Moneyball
 Nanos
 Nori
 ONNX

CHANGELOG.md

@@ -24,6 +24,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `created_time` and `last_updated_time` to `ml.get_model_group@200` ([#342](https://github.com/opensearch-project/opensearch-api-specification/pull/342))
 - Added spellcheck linter ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
 - Added tests for response payload ([#347](https://github.com/opensearch-project/opensearch-api-specification/pull/347))
+- Added support for testing `application/x-ndjson` payloads ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355))
 
 ### Changed
 
@@ -43,6 +44,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed `SlowlogThresholds` ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
 - Fixed `from_address` in notifications ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
 - Fixed `pages_processed` in rollups ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
+- Fixed `_bulk` spec request and response types ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355))
 
 ### Security
 

spec/namespaces/_core.yaml

@@ -2120,7 +2120,7 @@ components:
           schema:
             type: array
             items:
-              oneOf:
+              anyOf:
                 - $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer'
                 - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction'
                 - type: object

spec/schemas/_core.bulk.yaml

@@ -112,7 +112,7 @@ components:
       properties:
         _id:
           description: The document ID associated with the operation.
-          oneOf:
+          anyOf:
             - type: string
             - nullable: true
               type: string

tests/_core/bulk.yaml

@@ -0,0 +1,30 @@
+$schema: ../../json_schemas/test_story.schema.yaml
+
+skip: false
+description: Test bulk endpoint.
+epilogues:
+  - path: /books,movies
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Create an index.
+    path: /_bulk
+    method: POST
+    request_body:
+      content_type: application/x-ndjson
+      payload:
+        - {create: {_index: movies}}
+        - {director: Bennett Miller, title: Moneyball, year: 2011}
+  - synopsis: Bulk document CRUD.
+    path: /_bulk
+    method: POST
+    request_body:
+      content_type: application/x-ndjson
+      payload:
+        - {create: {_index: books, _id: book_1392214}}
+        - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960}
+        - {update: {_index: books, _id: book_1392214}}
+        - {doc: {pages: 376}}
+        - {update: {_index: books, _id: book_1392214}}
+        - {script: {source: 'ctx._source.pages = 376;'}}
+        - {delete: {_index: books, _id: book_1392214}}

tools/src/helpers.ts

@@ -97,6 +97,10 @@ export function to_json(content: any, replacer?: (this: any, key: string, value:
   return JSON.stringify(content, replacer, 2)
 }
 
+export function to_ndjson(content: any[]): string {
+  return _.join(_.map(content, JSON.stringify), "\n") + "\n"
+}
+
 export function write_json (file_path: string, content: any, replacer?: (this: any, key: string, value: any) => any): void {
   write_text(file_path, to_json(content, replacer))
 }

tools/src/tester/ChapterReader.ts

@@ -11,9 +11,8 @@ import { type ChapterRequest, type ActualResponse, type Parameter } from './type
 import { type OpenSearchHttpClient } from '../OpenSearchHttpClient'
 import { type StoryOutputs } from './StoryOutputs'
 import { Logger } from 'Logger'
-import { to_json } from '../helpers'
+import { to_json, to_ndjson } from '../helpers'
 
-// A lightweight client for testing the API
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
   private readonly logger: Logger
@@ -27,11 +26,16 @@ export default class ChapterReader {
     const response: Record<string, any> = {}
     const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
     const [url_path, params] = this.#parse_url(chapter.path, resolved_params)
-    const request_data = chapter.request_body?.payload !== undefined ? story_outputs.resolve_value(chapter.request_body.payload) : undefined
-    this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) | ${to_json(request_data)}`)
+    const content_type = chapter.request_body?.content_type ?? 'application/json'
+    const request_data = chapter.request_body?.payload !== undefined ? this.#serialize_payload(
+      story_outputs.resolve_value(chapter.request_body.payload),
+      content_type
+    ) : undefined
+    this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] | ${to_json(request_data)}`)
     await this._client.request({
       url: url_path,
       method: chapter.method,
+      headers: { 'Content-Type' : content_type },
       params,
       data: request_data
     }).then(r => {
@@ -54,6 +58,14 @@ export default class ChapterReader {
     return response as ActualResponse
   }
 
+  #serialize_payload(payload: any, content_type: string): any {
+    if (payload === undefined) return undefined
+    switch(content_type) {
+      case 'application/x-ndjson': return to_ndjson(payload as any[])
+      default: return payload
+    }
+  }
+
   #parse_url (path: string, parameters: Record<string, Parameter>): [string, Record<string, Parameter>] {
     const path_params = new Set<string>()
     const parsed_path = path.replace(/{(\w+)}/g, (_, key) => {

tools/src/tester/SchemaValidator.ts

@@ -35,6 +35,7 @@ export default class SchemaValidator {
     if (! valid) {
       this.logger.info(`# ${to_json(schema)}`)
       this.logger.info(`* ${to_json(data)}`)
+      this.logger.info(`& ${to_json(validate.errors)}`)
     }
     return {
       result: valid ? Result.PASSED : Result.FAILED,

tools/tests/helpers.test.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { sort_array_by_keys } from '../src/helpers'
+import { sort_array_by_keys, to_json, to_ndjson } from '../src/helpers'
 
 describe('helpers', () => {
   describe('sort_array_by_keys', () => {
@@ -25,4 +25,15 @@ describe('helpers', () => {
       expect(arr).toEqual(['GET', 'POST'])
     })
   })
+
+  test('to_json', () => {
+    expect(to_json({})).toEqual("{}")
+    expect(to_json({x: 1})).toEqual("{\n  \"x\": 1\n}")
+  })
+
+  test('to_ndjson', () => {
+    expect(to_ndjson([])).toEqual("\n")
+    expect(to_ndjson([{x: 1}])).toEqual("{\"x\":1}\n")
+    expect(to_ndjson([{x: 1}, {y: 'z'}])).toEqual("{\"x\":1}\n{\"y\":\"z\"}\n")
+  })
 })

tools/tests/tester/ChapterReader.test.ts

@@ -0,0 +1,106 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { Logger } from 'Logger';
+import { OpenSearchHttpClient } from 'OpenSearchHttpClient';
+import axios from 'axios';
+import ChapterReader from 'tester/ChapterReader';
+import { StoryOutputs } from 'tester/StoryOutputs';
+
+jest.mock('axios');
+const mocked_axios = axios as jest.Mocked<typeof axios>;
+
+describe('ChapterReader', () => {
+  var reader: ChapterReader
+
+  beforeEach(() => {
+    mocked_axios.create.mockReturnThis()
+
+    mocked_axios.request.mockResolvedValue({
+      status: 200,
+      headers: {
+        'content-type': 'application/json'
+      }
+    });
+
+    reader = new ChapterReader(new OpenSearchHttpClient(), new Logger())
+  })
+
+  afterEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('sends a GET request', async () => {
+    const result = await reader.read({
+      id: 'id',
+      path: 'path',
+      method: 'GET',
+      parameters: undefined,
+      request_body: undefined,
+      output: undefined
+    }, new StoryOutputs())
+
+    expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
+    expect(mocked_axios.request.mock.calls).toEqual([
+      [{ url: 'path', method: 'GET', headers: { 'Content-Type': 'application/json' }, params: {}, data: undefined }]
+    ])
+  })
+
+  it('resolves path parameters', async () => {
+    const result = await reader.read({
+      id: 'id',
+      path: '{index}/path',
+      method: 'GET',
+      parameters: { index: 'books' },
+      request_body: undefined,
+      output: undefined
+    }, new StoryOutputs())
+
+    expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
+    expect(mocked_axios.request.mock.calls).toEqual([
+      [{ url: 'books/path', method: 'GET', headers: { 'Content-Type': 'application/json' }, params: {}, data: undefined }]
+    ])
+  })
+
+  it('sends a POST request', async () => {
+    const result = await reader.read({
+      id: 'id',
+      path: 'path',
+      method: 'POST',
+      parameters: { 'x': 1 },
+      request_body: { payload: { "body": "present" } },
+      output: undefined
+    }, new StoryOutputs())
+
+    expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
+    expect(mocked_axios.request.mock.calls).toEqual([
+      [{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/json' }, params: { 'x': 1 }, data: { 'body': 'present' } }]
+    ])
+  })
+
+  it('sends an nd-json POST request', async () => {
+    const result = await reader.read({
+      id: 'id',
+      path: 'path',
+      method: 'POST',
+      parameters: { 'x': 1 },
+      request_body: {
+        content_type: 'application/x-ndjson',
+        payload: [{ "body": "present" }]
+      },
+      output: undefined
+    }, new StoryOutputs())
+
+    expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined })
+    expect(mocked_axios.request.mock.calls).toEqual([
+      [{ url: 'path', method: 'POST', headers: { 'Content-Type': 'application/x-ndjson' }, params: { 'x': 1 }, data: "{\"body\":\"present\"}\n"}]
+    ])
+  })
+})
+