Check response payload.

CHANGELOG.md

@@ -23,6 +23,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `distribution` field to `OpenSearchVersionInfo` ([#336](https://github.com/opensearch-project/opensearch-api-specification/pull/336)) 
 - Added `created_time` and `last_updated_time` to `ml.get_model_group@200` ([#342](https://github.com/opensearch-project/opensearch-api-specification/pull/342))
 - Added spellcheck linter ([#341](https://github.com/opensearch-project/opensearch-api-specification/pull/341))
+- Added tests for response payload ([#347](https://github.com/opensearch-project/opensearch-api-specification/pull/347))
 
 ### Changed
 

package.json

@@ -48,6 +48,7 @@
     "eslint-plugin-yml": "^1.14.0",
     "globals": "^15.0.0",
     "jest": "^29.7.0",
+    "json-diff-ts": "^4.0.1",
     "json-schema-to-typescript": "^14.0.4",
     "ts-jest": "^29.1.2"
   }

tests/_core/info.yaml

@@ -16,3 +16,7 @@ chapters:
       pretty: false
     response:
       status: 200
+      payload:
+        version:
+          distribution: opensearch
+        tagline: 'The OpenSearch Project: https://opensearch.org/'

tools/src/tester/ChapterEvaluator.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type Chapter, type ActualResponse } from './types/story.types'
+import { type Chapter, type ActualResponse, type Payload } from './types/story.types'
 import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types'
 import { type ParsedOperation } from './types/spec.types'
 import { overall_result } from './helpers'
@@ -16,6 +16,8 @@ import type OperationLocator from './OperationLocator'
 import type SchemaValidator from './SchemaValidator'
 import { type StoryOutputs } from './StoryOutputs'
 import { ChapterOutput } from './ChapterOutput'
+import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
+import _ from 'lodash'
 
 export default class ChapterEvaluator {
   private readonly _operation_locator: OperationLocator
@@ -36,13 +38,20 @@ export default class ChapterEvaluator {
     const params = this.#evaluate_parameters(chapter, operation)
     const request_body = this.#evaluate_request_body(chapter, operation)
     const status = this.#evaluate_status(chapter, response)
-    const payload = status.result === Result.PASSED ? this.#evaluate_payload(response, operation) : { result: Result.SKIPPED }
+    const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED }
+    const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(response, operation) : { result: Result.SKIPPED }
     const output_values = ChapterOutput.extract_output_values(response, chapter.output)
     return {
       title: chapter.synopsis,
-      overall: { result: overall_result(Object.values(params).concat([request_body, status, payload]).concat(output_values ? [output_values] : [])) },
+      overall: { result: overall_result(Object.values(params).concat([
+        request_body, status, payload_body_evaluation, payload_schema_evaluation
+      ]).concat(output_values ? [output_values] : [])) },
       request: { parameters: params, request_body },
-      response: { status, payload },
+      response: {
+        status,
+        payload_body: payload_body_evaluation,
+        payload_schema: payload_schema_evaluation
+      },
       ...(output_values ? { output_values } : {})
     }
   }
@@ -74,7 +83,21 @@ export default class ChapterEvaluator {
     }
   }
 
-  #evaluate_payload(response: ActualResponse, operation: ParsedOperation): Evaluation {
+  #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {
+    if (expected_payload == null) return { result: Result.PASSED }
+    const delta = atomizeChangeset(diff(expected_payload, response.payload))
+    const messages: string[] = _.compact(delta.map((value, _index, _array) => {
+      switch(value.type) {
+        case Operation.UPDATE:
+          return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
+        case Operation.REMOVE:
+          return `missing ${value.path.replace('$.', '')}='${value.value}'`
+      }
+    }))
+    return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ')} : { result: Result.PASSED }
+  }
+
+  #evaluate_payload_schema(response: ActualResponse, operation: ParsedOperation): Evaluation {
     const content_type = response.content_type ?? 'application/json'
     const content = operation.responses[response.status]?.content[content_type]
     const schema = content?.schema

tools/src/tester/ResultLogger.ts

@@ -53,7 +53,8 @@ export class ConsoleResultLogger implements ResultLogger {
     this.#log_parameters(chapter.request?.parameters ?? {})
     this.#log_request_body(chapter.request?.request_body)
     this.#log_status(chapter.response?.status)
-    this.#log_payload(chapter.response?.payload)
+    this.#log_payload_body(chapter.response?.payload_body)
+    this.#log_payload_schema(chapter.response?.payload_schema)
   }
 
   #log_parameters (parameters: Record<string, Evaluation>): void {
@@ -75,9 +76,14 @@ export class ConsoleResultLogger implements ResultLogger {
     this.#log_evaluation(evaluation, 'RESPONSE STATUS', this._tab_width * 3)
   }
 
-  #log_payload (evaluation: Evaluation | undefined): void {
+  #log_payload_body (evaluation: Evaluation | undefined): void {
     if (evaluation == null) return
-    this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD', this._tab_width * 3)
+    this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD BODY', this._tab_width * 3)
+  }
+
+  #log_payload_schema (evaluation: Evaluation | undefined): void {
+    if (evaluation == null) return
+    this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD SCHEMA', this._tab_width * 3)
   }
 
   #log_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void {

tools/src/tester/types/eval.types.ts

@@ -32,7 +32,8 @@ export interface ChapterEvaluation {
   }
   response?: {
     status: Evaluation
-    payload: Evaluation
+    payload_body: Evaluation,
+    payload_schema: Evaluation
   }
   output_values?: EvaluationWithOutput
 }

tools/tests/tester/fixtures/evals/error/chapter_error.yaml

@@ -27,7 +27,9 @@ chapters:
         message: 'Expected status 200, but received 404: application/json. no such index
           [undefined]'
         error: Request failed with status code 404
-      payload:
+      payload_body:
+        result: SKIPPED
+      payload_schema:
         result: SKIPPED
   - title: This chapter should be skipped.
     overall:

tools/tests/tester/fixtures/evals/failed/invalid_data.yaml

@@ -20,7 +20,9 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: PASSED
   - title: This chapter should fail because the request body is invalid.
     overall:
@@ -35,9 +37,11 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: PASSED
-  - title: This chapter should fail because the response is invalid.
+  - title: This chapter should fail because the response data and schema are invalid.
     overall:
       result: FAILED
     request:
@@ -49,7 +53,10 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: FAILED
+        message: expected acknowledged='false', got 'true', missing shards_acknowledged='true'
+      payload_schema:
         result: FAILED
         message: data must NOT have additional properties
 

tools/tests/tester/fixtures/evals/failed/not_found.yaml

@@ -26,7 +26,9 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: PASSED
   - title: This chapter should fail because the request body is not defined in the spec.
     overall:
@@ -41,7 +43,9 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: PASSED
   - title: This chapter should fail because the response is not defined in the spec.
     overall:
@@ -55,7 +59,9 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: FAILED
         message: 'Schema for "404: application/json" response not found in the spec.'
 

tools/tests/tester/fixtures/evals/passed.yaml

@@ -19,7 +19,9 @@ chapters:
     response:
       status:
         result: PASSED
-      payload:
+      payload_body:
+        result: PASSED
+      payload_schema:
         result: PASSED
 
 epilogues:

tools/tests/tester/fixtures/stories/failed/invalid_data.yaml

@@ -22,12 +22,13 @@ chapters:
     request_body:
       payload:
         aliases: {}
-  - synopsis: This chapter should fail because the response is invalid.
+  - synopsis: This chapter should fail because the response data and schema are invalid.
     path: /{index}
     method: DELETE
     parameters:
       index: books
     response:
       status: 200
       payload:
-        shards_acknowledged: true
+        acknowledged: false
+        shards_acknowledged: true

tools/tests/tester/helpers.ts

@@ -90,7 +90,8 @@ export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation {
       response: c.response !== undefined
         ? {
           status: flatten(c.response.status),
-          payload: flatten(c.response.payload)
+          payload_body: flatten(c.response.payload_body),
+          payload_schema: flatten(c.response.payload_schema)
         }
         : undefined
     })) as T