Merge branch 'main' into querygroup-api

Signed-off-by: Ruirui Zhang <[email protected]>

.cspell

@@ -19,7 +19,9 @@ Kuromoji
 Lovins
 Lucene
 Millis
+Moneyball
 Multisearch
+Moneyball
 Nanos
 Nori
 ONNX

CHANGELOG.md

@@ -24,6 +24,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `created_time` and `last_updated_time` to `ml.get_model_group@200` ([#342](https://github.com/opensearch-project/opensearch-api-specification/pull/342))
 - Added spellcheck linter ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
 - Added tests for response payload ([#347](https://github.com/opensearch-project/opensearch-api-specification/pull/347))
+- Added `cancel_after_time_interval` and `phase_took` in `_search` ([#353](https://github.com/opensearch-project/opensearch-api-specification/pull/353))
+- Added support for testing `application/x-ndjson` payloads ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355))
 - Added QueryGroup API documentation ([#356](https://github.com/opensearch-project/opensearch-api-specification/pull/356))
 
 ### Changed
@@ -37,13 +39,17 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Removed
 
+- Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358)
+
 ### Fixed
 
 - Fixed GitHub pages ([#215](https://github.com/opensearch-project/opensearch-api-specification/pull/215))
 - Fixed missing 201 response in `/{index}/_doc/{id}` ([#331](https://github.com/opensearch-project/opensearch-api-specification/pull/331))
 - Fixed `SlowlogThresholds` ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
-- Fixed `from_address` in notifications ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
-- Fixed `pages_processed` in rollups ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
+- Fixed `from_address` in `SmtpAccount` ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
+- Fixed `pages_processed` in `/_plugins/_rollup` ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
+- Fixed `_bulk` spec request and response types ([#355](https://github.com/opensearch-project/opensearch-api-specification/pull/355))
+- Fixed `text/plain` response in `/_cat` ([#357](https://github.com/opensearch-project/opensearch-api-specification/pull/357))
 
 ### Security
 

DEVELOPER_GUIDE.md

@@ -167,7 +167,6 @@ Below is the simplified version of the test story that tests the [index operatio
 ```yaml
 $schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
 
-skip: false # Skip this test story if set to true.
 description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
 
 prologues: [] # No prologues are needed for this story.

json_schemas/test_story.schema.yaml

@@ -4,10 +4,6 @@ type: object
 properties:
   $schema:
     type: string
-  skip:
-    type: boolean
-    description: If true, the story will be skipped.
-    default: false
   description:
     type: string
   prologues:

spec/namespaces/cat.yaml

@@ -12,9 +12,6 @@ paths:
       description: Returns help for the Cat APIs.
       externalDocs:
         url: https://opensearch.org/docs/latest/api-reference/cat/index/
-      parameters:
-        - $ref: '#/components/parameters/cat.help::query.help'
-        - $ref: '#/components/parameters/cat.help::query.s'
       responses:
         '200':
           $ref: '#/components/responses/cat.help@200'
@@ -842,11 +839,9 @@ components:
     cat.help@200:
       description: ''
       content:
-        application/json:
+        text/plain:
           schema:
-            type: array
-            items:
-              $ref: '../schemas/cat.help.yaml#/components/schemas/HelpRecord'
+            type: string
     cat.indices@200:
       description: ''
       content:
@@ -1432,25 +1427,6 @@ components:
         type: boolean
         default: false
         description: Verbose mode. Display column headers.
-    cat.help::query.help:
-      name: help
-      in: query
-      description: Return help information.
-      schema:
-        type: boolean
-        default: false
-        description: Return help information.
-    cat.help::query.s:
-      name: s
-      in: query
-      description: Comma-separated list of column names or column aliases to sort by.
-      style: form
-      schema:
-        type: array
-        items:
-          type: string
-        description: Comma-separated list of column names or column aliases to sort by.
-      explode: true
     cat.indices::path.index:
       in: path
       name: index

spec/schemas/_common.yaml

@@ -1820,3 +1820,25 @@ components:
         - minimum_index_compatibility_version
         - minimum_wire_compatibility_version
         - number
+    PhaseTook:
+      type: object
+      properties:
+        dfs_pre_query:
+          $ref: '#/components/schemas/uint'
+        query:
+          $ref: '#/components/schemas/uint'
+        fetch:
+          $ref: '#/components/schemas/uint'
+        dfs_query:
+          $ref: '#/components/schemas/uint'
+        expand:
+          $ref: '#/components/schemas/uint'
+        can_match:
+          $ref: '#/components/schemas/uint'
+      required:
+        - dfs_pre_query
+        - query
+        - fetch
+        - dfs_query
+        - expand
+        - can_match

spec/schemas/_core.bulk.yaml

@@ -112,7 +112,7 @@ components:
       properties:
         _id:
           description: The document ID associated with the operation.
-          oneOf:
+          anyOf:
             - type: string
             - nullable: true
               type: string

spec/schemas/_core.search.yaml

@@ -851,6 +851,8 @@ components:
           type: boolean
         _shards:
           $ref: '_common.yaml#/components/schemas/ShardStatistics'
+        phase_took:
+          $ref: '_common.yaml#/components/schemas/PhaseTook'
         hits:
           $ref: '#/components/schemas/HitsMetadata'
         aggregations:

tests/_core/bulk.yaml

@@ -0,0 +1,30 @@
+$schema: ../../json_schemas/test_story.schema.yaml
+
+skip: false
+description: Test bulk endpoint.
+epilogues:
+  - path: /books,movies
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Create an index.
+    path: /_bulk
+    method: POST
+    request_body:
+      content_type: application/x-ndjson
+      payload:
+        - {create: {_index: movies}}
+        - {director: Bennett Miller, title: Moneyball, year: 2011}
+  - synopsis: Bulk document CRUD.
+    path: /_bulk
+    method: POST
+    request_body:
+      content_type: application/x-ndjson
+      payload:
+        - {create: {_index: books, _id: book_1392214}}
+        - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960}
+        - {update: {_index: books, _id: book_1392214}}
+        - {doc: {pages: 376}}
+        - {update: {_index: books, _id: book_1392214}}
+        - {script: {source: 'ctx._source.pages = 376;'}}
+        - {delete: {_index: books, _id: book_1392214}}

tests/_core/info.yaml

@@ -1,7 +1,6 @@
 
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test root endpoint.
 chapters:
   - synopsis: Get server info.

tests/_core/search.yaml

@@ -0,0 +1,100 @@
+$schema: ../../json_schemas/test_story.schema.yaml
+
+skip: false
+description: Test search endpoint.
+prologues:
+  - path: /movies/_doc
+    method: POST
+    parameters:
+      refresh: true
+    request_body:
+      payload:
+        director: Bennett Miller
+        title: Moneyball
+        year: 2011
+    response:
+      status: 201
+epilogues:
+  - path: /movies
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Search with size=0.
+    path: /{index}/_search
+    parameters:
+      index: movies
+    method: POST
+    request_body:
+      payload:
+        size: 0
+    response:
+      status: 200
+      payload:
+        timed_out: false
+        hits:
+          total:
+            value: 1
+            relation: eq
+          max_score: null
+          hits: []
+  - synopsis: Search with size=1.
+    path: /{index}/_search
+    parameters:
+      index: movies
+    method: POST
+    request_body:
+      payload:
+        size: 1
+        query:
+          match_all: {}
+    response:
+      status: 200
+      payload:
+        timed_out: false
+        hits:
+          total:
+            value: 1
+            relation: eq
+          max_score: 1
+          hits:
+            - _index: movies
+              _score: 1
+              _source:
+                director: Bennett Miller
+                title: Moneyball
+                year: 2011
+  - synopsis: Search with multi_match query.
+    path: /{index}/_search
+    parameters:
+      index: movies
+    method: POST
+    request_body:
+      payload:
+        size: 1
+        query:
+          multi_match:
+            query: miller
+            fields: ["title^2", "director"]
+    response:
+      status: 200
+      payload:
+        timed_out: false
+        hits:
+          total:
+            value: 1
+            relation: eq
+          hits:
+            - _index: movies
+              _source:
+                director: Bennett Miller
+                title: Moneyball
+                year: 2011
+  - synopsis: Search with parameters.
+    path: /{index}/_search
+    parameters:
+      index: movies
+      cancel_after_time_interval: 10s
+      phase_took: true
+    method: POST
+    response:
+      status: 200

tests/cat/index.yaml

@@ -0,0 +1,11 @@
+$schema: ../../json_schemas/test_story.schema.yaml
+
+skip: false
+description: Test cat endpoints.
+chapters:
+  - synopsis: Cat with a json response.
+    path: /_cat
+    method: GET
+    response:
+      status: 200
+      content_type: text/plain

tests/indices/_doc.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test inserting and retrieving a doc.
 epilogues:
   - path: /movies

tests/indices/index.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test endpoints relevant the lifecycle of an index, from creation to deletion.
 epilogues:
   - path: /books

tests/ingest/pipeline.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: |
   Test the creation of an ingest pipeline with a text embedding processor.
 epilogues:

tests/ml/model_groups.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: |
   Test the creation of model groups.
 epilogues:

tools/src/helpers.ts

@@ -97,6 +97,10 @@ export function to_json(content: any, replacer?: (this: any, key: string, value:
   return JSON.stringify(content, replacer, 2)
 }
 
+export function to_ndjson(content: any[]): string {
+  return _.join(_.map(content, JSON.stringify), "\n") + "\n"
+}
+
 export function write_json (file_path: string, content: any, replacer?: (this: any, key: string, value: any) => any): void {
   write_text(file_path, to_json(content, replacer))
 }

tools/src/merger/OpenApiMerger.ts

@@ -69,7 +69,7 @@ export default class OpenApiMerger {
 
   // Redirect schema references in namespace files to local references in single-file spec.
   redirect_refs_in_namespace (obj: any): void {
-    const ref: string = obj.$ref
+    const ref: string = obj?.$ref
     if (ref?.startsWith('../schemas/')) { obj.$ref = ref.replace('../schemas/', '#/components/schemas/').replace('.yaml#/components/schemas/', ':') }
 
     for (const key in obj) {