Merge remote-tracking branch 'upstream/main' into security-api-spec

CLIENT_GENERATOR_GUIDE.md

@@ -45,7 +45,9 @@ def search(self, index=None, body=None):
 You will also encounter `x-overloaded-param: metric` for the `node_id` path parameter of the `GET /_nodes/{node_id}` operation in `nodes.info` action. This is a special case where the path parameter is overloaded to accept either a node ID or a metric name. When the user evokes the `client.nodes.info` method with either `metric` or `node_id` (but not both), the method will use the `GET /_nodes/{node_id}` operation. When evoked with both `metric` and `node_id`, it will use the `GET /_nodes/{node_id}/{metric}` operation.
 
 ## Handling Bulk Operations
-Some operations accept a bulk of data in the request body. For example, the `bulk` action accepts a bulk of index, update, and delete operations on multiple documents. Unlike other operations where the request body is a **JSON object**, the request body for bulk operations is an **NDJSON** (i.e a [Newline-delimited JSON](https://github.com/ndjson/ndjson-spec)). When encountering this type of operation, the client must serialize the request body accordingly, and set the `Content-Type` header to `application/x-ndjson`.
+Some operations accept a bulk of data in the request body. For example, the `bulk` action accepts a bulk of index, update, and delete operations on multiple documents. Unlike other operations where the request body is a **JSON object**, the request body for bulk operations is an **NDJSON** (i.e a [Newline-delimited JSON](https://github.com/ndjson/ndjson-spec)). 
+
+In the spec, these request bodies have the `content` of `application/x-ndjson` and their schemas are of type array. In this situation, the client must serialize the request body accordingly, and set the `Content-Type` header to `application/x-ndjson`.
 
 ## Parameter Validation
 As of right now, most clients only validate whether required parameters are present. The clients do not validate the values of parameters against the enum values or regex patterns. This is to reduce performance overhead for the clients as the validation is already done on the server. However, the list of enum values and regex patterns are often written into the parameter description.

json_schemas/_superseded_operations.yaml

@@ -0,0 +1,20 @@
+$schema: http://json-schema.org/draft-07/schema#
+type: object
+patternProperties:
+  ^\$schema$:
+    type: string
+  ^/:
+    type: object
+    properties:
+      superseded_by:
+        type: string
+        pattern: ^/
+      operations:
+        type: array
+        items:
+          type: string
+          enum: [GET, POST, PUT, DELETE, HEAD, OPTIONS, PATCH]
+    required: [superseded_by, operations]
+    additionalProperties: false
+required: [$schema]
+additionalProperties: false

spec/_superseded_operations.yaml

@@ -1,3 +1,5 @@
+$schema: ../json_schemas/_superseded_operations.yaml
+
 /_opendistro/_alerting/destinations:
   superseded_by: /_plugins/_alerting/destinations
   operations:

spec/namespaces/_core.yaml

@@ -2110,7 +2110,7 @@ components:
   requestBodies:
     bulk:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
@@ -2119,7 +2119,6 @@ components:
                 - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction'
                 - type: object
             description: The operation definition and data (action-data pairs), separated by newlines
-            x-serialize: bulk
       required: true
     clear_scroll:
       content:
@@ -2220,23 +2219,21 @@ components:
       required: true
     msearch:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/_core.msearch.yaml#/components/schemas/RequestItem'
             description: The request definitions (metadata-search request definition pairs), separated by newlines
-            x-serialize: bulk
       required: true
     msearch_template:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/_core.msearch_template.yaml#/components/schemas/RequestItem'
             description: The request definitions (metadata-search request definition pairs), separated by newlines
-            x-serialize: bulk
       required: true
     mtermvectors:
       content:

spec/namespaces/search_pipeline.yaml

@@ -141,4 +141,4 @@ components:
       in: query
       description: Operation timeout.
       schema:
-        $ref: '../schemas/_common.yaml#/components/schemas/Duration'
+        $ref: '../schemas/_common.yaml#/components/schemas/Duration'

spec/namespaces/security.yaml

@@ -1084,103 +1084,103 @@ components:
       required: false
     security.patch_action_group:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_action_groups:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_audit_configuration:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_configuration:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_distinguished_names:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_role:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_role_mapping:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_role_mappings:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_roles:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_tenant:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_tenants:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_user:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:
               $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
       required: true
     security.patch_users:
       content:
-        application/json:
+        application/x-ndjson:
           schema:
             type: array
             items:

tools/helpers.ts

@@ -2,7 +2,7 @@ import fs from 'fs'
 import YAML from 'yaml'
 import _ from 'lodash'
 
-export function resolve (ref: string, root: Record<string, any>) {
+export function resolveRef (ref: string, root: Record<string, any>): Record<string, any> | undefined {
   const paths = ref.replace('#/', '').split('/')
   for (const p of paths) {
     root = root[p]
@@ -11,6 +11,21 @@ export function resolve (ref: string, root: Record<string, any>) {
   return root
 }
 
+export function resolveObj (obj: Record<string, any> | undefined, root: Record<string, any>) {
+  if (obj === undefined) return undefined
+  if (obj.$ref) return resolveRef(obj.$ref, root)
+  return obj
+}
+
+export function dig (obj: Record<string, any>, path: string[], root: Record<string, any>): any {
+  let value = obj
+  for (const p of path) {
+    value = resolveObj(value, root)?.[p]
+    if (value === undefined) break
+  }
+  return value
+}
+
 export function sortByKey (obj: Record<string, any>, priorities: string[] = []) {
   const orders = _.fromPairs(priorities.map((k, i) => [k, i + 1]))
   const sorted = _.entries(obj).sort((a, b) => {

tools/linter/SpecValidator.ts

@@ -4,16 +4,19 @@ import RootFile from './components/RootFile'
 import { type ValidationError } from '../types'
 import PathRefsValidator from './PathRefsValidator'
 import SchemaRefsValidator from './SchemaRefsValidator'
+import SupersededOperationsFile from './components/SupersededOperationsFile'
 
 export default class SpecValidator {
   root_file: RootFile
+  superseded_ops_files: SupersededOperationsFile
   namespaces_folder: NamespacesFolder
   schemas_folder: SchemasFolder
   path_refs_validator: PathRefsValidator
   schema_refs_validator: SchemaRefsValidator
 
   constructor (root_folder: string) {
     this.root_file = new RootFile(`${root_folder}/opensearch-openapi.yaml`)
+    this.superseded_ops_files = new SupersededOperationsFile(`${root_folder}/_superseded_operations.yaml`)
     this.namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`)
     this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`)
     this.path_refs_validator = new PathRefsValidator(this.root_file, this.namespaces_folder)
@@ -26,11 +29,12 @@ export default class SpecValidator {
       ...this.namespaces_folder.validate(),
       ...this.schemas_folder.validate()
     ]
-    if (component_errors.length) return component_errors
+    if (component_errors.length > 0) return component_errors
 
     return [
       ...this.path_refs_validator.validate(),
-      ...this.schema_refs_validator.validate()
+      ...this.schema_refs_validator.validate(),
+      ...this.superseded_ops_files.validate()
     ]
   }
 }

tools/linter/components/NamespaceFile.ts

@@ -3,7 +3,7 @@ import { type OperationSpec, type ValidationError } from '../../types'
 import OperationGroup from './OperationGroup'
 import _ from 'lodash'
 import Operation from './Operation'
-import { resolve } from '../../helpers'
+import { resolveRef } from '../../helpers'
 import FileValidator from './base/FileValidator'
 
 const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace']
@@ -68,7 +68,7 @@ export default class NamespaceFile extends FileValidator {
 
   validate_unresolved_refs (): ValidationError[] {
     return Array.from(this.refs()).map((ref) => {
-      if (resolve(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref)
+      if (resolveRef(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref)
     }).filter((e) => e) as ValidationError[]
   }
 

tools/linter/components/SupersededOperationsFile.ts

@@ -0,0 +1,23 @@
+import FileValidator from './base/FileValidator'
+import ajv from 'ajv'
+import fs from 'fs'
+import YAML from 'yaml'
+import { type ValidationError } from '../../types'
+
+export default class SupersededOperationsFile extends FileValidator {
+  JSON_SCHEMA_PATH = '../json_schemas/_superseded_operations.yaml'
+
+  validate (): ValidationError[] {
+    return [
+      this.validate_json_schema()
+    ].filter(e => e) as ValidationError[]
+  }
+
+  validate_json_schema (): ValidationError | undefined {
+    const schema = YAML.parse(fs.readFileSync(this.JSON_SCHEMA_PATH, 'utf8'))
+    const validator = (new ajv()).compile(schema)
+    if (!validator(this.spec())) {
+      return this.error(`File content does not match JSON schema found in '${this.JSON_SCHEMA_PATH}':\n ${JSON.stringify(validator.errors, null, 2)}`)
+    }
+  }
+}

tools/merger/SupersededOpsGenerator.ts

@@ -9,6 +9,7 @@ export default class SupersededOpsGenerator {
   constructor (root_path: string) {
     const file_path = root_path + '/_superseded_operations.yaml'
     this.superseded_ops = YAML.parse(fs.readFileSync(file_path, 'utf8'))
+    delete this.superseded_ops.$schema
   }
 
   generate (spec: Record<string, any>): void {

tools/package.json

@@ -11,10 +11,12 @@
     "test": "jest"
   },
   "dependencies": {
+    "ajv": "^8.13.0",
     "@apidevtools/swagger-parser": "^10.1.0",
     "@types/lodash": "^4.14.202",
     "@types/node": "^20.10.3",
     "lodash": "^4.17.21",
+    "typescript": "^5.4.5",
     "ts-node": "^10.9.1",
     "yaml": "^2.3.4"
   },
@@ -30,7 +32,6 @@
     "eslint-plugin-promise": "^6.1.1",
     "globals": "^15.0.0",
     "jest": "^29.7.0",
-    "ts-jest": "^29.1.2",
-    "typescript": "^5.4.5"
+    "ts-jest": "^29.1.2"
   }
 }

tools/test/linter/SupersededOperationsFile.test.ts

@@ -0,0 +1,11 @@
+import SupersededOperationsFile from '../../linter/components/SupersededOperationsFile'
+
+test('validate()', () => {
+  const validator = new SupersededOperationsFile('./test/linter/fixtures/_superseded_operations.yaml')
+  expect(validator.validate()).toEqual([
+    {
+      file: 'fixtures/_superseded_operations.yaml',
+      message: "File content does not match JSON schema found in '../json_schemas/_superseded_operations.yaml':\n [\n  {\n    \"instancePath\": \"/~1hello~1world/operations/1\",\n    \"schemaPath\": \"#/patternProperties/%5E~1/properties/operations/items/enum\",\n    \"keyword\": \"enum\",\n    \"params\": {\n      \"allowedValues\": [\n        \"GET\",\n        \"POST\",\n        \"PUT\",\n        \"DELETE\",\n        \"HEAD\",\n        \"OPTIONS\",\n        \"PATCH\"\n      ]\n    },\n    \"message\": \"must be equal to one of the allowed values\"\n  }\n]"
+    }
+  ])
+})