Validate response content types (#394)

* Validate response content types Signed-off-by: Thomas Farr <[email protected]> * Correct cat health test Signed-off-by: Thomas Farr <[email protected]> * Correct passed test fixture Signed-off-by: Thomas Farr <[email protected]> * Add test for mismatched content-type Signed-off-by: Thomas Farr <[email protected]> * Add cat.indices to test fixture excerpt schema Signed-off-by: Thomas Farr <[email protected]> --------- Signed-off-by: Thomas Farr <[email protected]>
opensearch-project · Jul 10, 2024 · 45953f0 · 45953f0
1 parent 046b7d1
commit 45953f0
Show file tree

Hide file tree

Showing 8 changed files with 139 additions and 20 deletions.
diff --git a/tests/cat/health.yaml b/tests/cat/health.yaml
@@ -76,7 +76,7 @@ chapters:
       format: cbor
     response:
       status: 200
-      content_type: application/yaml
+      content_type: application/cbor
       payload:
         - node.total: '1'
           status: yellow

diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts
@@ -20,6 +20,7 @@ import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
 import _ from 'lodash'
 import { Logger } from 'Logger'
 import { to_json } from '../helpers'
+import { APPLICATION_JSON } from "./MimeTypes";
 
 export default class ChapterEvaluator {
   private readonly logger: Logger
@@ -43,7 +44,7 @@ export default class ChapterEvaluator {
     const request_body = this.#evaluate_request_body(chapter, operation)
     const status = this.#evaluate_status(chapter, response)
     const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED }
-    const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(response, operation) : { result: Result.SKIPPED }
+    const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
     const output_values = ChapterOutput.extract_output_values(response, chapter.output)
     return {
       title: chapter.synopsis,
@@ -71,7 +72,7 @@ export default class ChapterEvaluator {
 
   #evaluate_request_body(chapter: Chapter, operation: ParsedOperation): Evaluation {
     if (!chapter.request_body) return { result: Result.PASSED }
-    const content_type = chapter.request_body.content_type ?? 'application/json'
+    const content_type = chapter.request_body.content_type ?? APPLICATION_JSON
     const schema = operation.requestBody?.content[content_type]?.schema
     if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` }
     return this._schema_validator.validate(schema, chapter.request_body?.payload ?? {})
@@ -81,13 +82,13 @@ export default class ChapterEvaluator {
     const expected_status = chapter.response?.status ?? 200
     if (response.status === expected_status) return { result: Result.PASSED }
 
-    var result: Evaluation = {
+    const result: Evaluation = {
       result: Result.ERROR,
       message: _.join(_.compact([
         `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`,
         response.message
       ]), ' ')
-    }
+    };
 
     if (response.error !== undefined) {
       result.error = response.error as Error
@@ -112,14 +113,27 @@ export default class ChapterEvaluator {
     return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
   }
 
-  #evaluate_payload_schema(response: ActualResponse, operation: ParsedOperation): Evaluation {
-    const content_type = response.content_type ?? 'application/json'
-    const content = operation.responses[response.status]?.content
-    // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
-    const content_type_content = content ? content[content_type] : undefined
-    const schema = content_type_content?.schema
-    if (schema == null && content != null) return { result: Result.PASSED }
-    if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` }
-    return this._schema_validator.validate(schema, response.payload)
+  #evaluate_payload_schema(chapter: Chapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
+    const content_type = chapter.response?.content_type ?? APPLICATION_JSON
+
+    if (response.content_type !== content_type) {
+      return {
+        result: Result.FAILED,
+        message: `Expected content type ${content_type}, but received ${response.content_type}.`
+      }
+    }
+
+    const content = operation.responses[response.status]?.content?.[content_type]
+
+    if (content == null) {
+      return {
+        result: Result.FAILED,
+        message: `Schema for "${response.status}: ${content_type}" response not found in the spec.`
+      }
+    }
+
+    if (content.schema == null) return { result: Result.PASSED }
+
+    return this._schema_validator.validate(content.schema, response.payload)
   }
 }
diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts
@@ -16,6 +16,7 @@ import qs from 'qs'
 import YAML from 'yaml'
 import CBOR from 'cbor'
 import SMILE from 'smile-js'
+import { APPLICATION_CBOR, APPLICATION_JSON, APPLICATION_SMILE, APPLICATION_YAML, TEXT_PLAIN } from "./MimeTypes";
 
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
@@ -30,7 +31,7 @@ export default class ChapterReader {
     const response: Record<string, any> = {}
     const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
     const [url_path, params] = this.#parse_url(chapter.path, resolved_params)
-    const content_type = chapter.request_body?.content_type ?? 'application/json'
+    const content_type = chapter.request_body?.content_type ?? APPLICATION_JSON
     const request_data = chapter.request_body?.payload !== undefined ? this.#serialize_payload(
       story_outputs.resolve_value(chapter.request_body.payload),
       content_type
@@ -90,11 +91,11 @@ export default class ChapterReader {
     if (content_type === undefined) return payload
     const payload_buffer = Buffer.from(payload as string, 'binary')
     switch (content_type) {
-      case 'text/plain': return payload_buffer.toString()
-      case 'application/json': return payload.length == 0 ? {} : JSON.parse(payload_buffer.toString())
-      case 'application/yaml': return payload.length == 0 ? {} : YAML.parse(payload_buffer.toString())
-      case 'application/cbor': return payload.length == 0 ? {} : CBOR.decode(payload_buffer)
-      case 'application/smile': return payload.length == 0 ? {} : SMILE.parse(payload_buffer)
+      case TEXT_PLAIN: return payload_buffer.toString()
+      case APPLICATION_JSON: return payload.length == 0 ? {} : JSON.parse(payload_buffer.toString())
+      case APPLICATION_YAML: return payload.length == 0 ? {} : YAML.parse(payload_buffer.toString())
+      case APPLICATION_CBOR: return payload.length == 0 ? {} : CBOR.decode(payload_buffer)
+      case APPLICATION_SMILE: return payload.length == 0 ? {} : SMILE.parse(payload_buffer)
       default: return payload_buffer.toString()
     }
   }

diff --git a/tools/src/tester/MimeTypes.ts b/tools/src/tester/MimeTypes.ts
@@ -0,0 +1,14 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+export const APPLICATION_CBOR = 'application/cbor'
+export const APPLICATION_JSON = 'application/json'
+export const APPLICATION_SMILE = 'application/smile'
+export const APPLICATION_YAML = 'application/yaml'
+export const TEXT_PLAIN = 'text/plain'
diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml
@@ -41,6 +41,25 @@ chapters:
         result: PASSED
       payload_schema:
         result: PASSED
+  - title: This chapter should fail because the response content type does not match.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        format:
+          result: PASSED
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload_body:
+        result: PASSED
+      payload_schema:
+        result: FAILED
+        message: 'Expected content type application/json, but received application/yaml.'
   - title: This chapter should fail because the response data and schema are invalid.
     overall:
       result: FAILED

diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml
@@ -25,6 +25,20 @@ paths:
       responses:
         '200':
           $ref: '#/components/responses/cat.health@200'
+  /_cat/indices/{index}:
+    get:
+      operationId: cat.indices.1
+      x-operation-group: cat.indices
+      x-version-added: '1.0'
+      description: 'Returns information about indices: number of primaries and replicas, document counts, disk size, ...'
+      externalDocs:
+        url: https://opensearch.org/docs/latest/api-reference/cat/cat-indices/
+      parameters:
+        - $ref: '#/components/parameters/cat.indices::path.index'
+        - $ref: '#/components/parameters/cat.indices::query.format'
+      responses:
+        '200':
+          $ref: '#/components/responses/cat.indices@200'
   /{index}:
     delete:
       operationId: indices.delete.0
@@ -111,6 +125,32 @@ components:
             type: array
             items:
               type: object
+    cat.indices@200:
+      description: ''
+      content:
+        text/plain:
+          schema:
+            type: string
+        application/json:
+          schema:
+            type: array
+            items:
+              type: object
+        application/yaml:
+          schema:
+            type: array
+            items:
+              type: object
+        application/cbor:
+          schema:
+            type: array
+            items:
+              type: object
+        application/smile:
+          schema:
+            type: array
+            items:
+              type: object
     indices.delete@200:
       description: ''
       content:
@@ -144,6 +184,26 @@ components:
       name: format
       schema:
         type: string
+    cat.help::query.format:
+      in: query
+      name: format
+      schema:
+        type: string
+    cat.indices::path.index:
+      in: path
+      name: index
+      description: |-
+        Comma-separated list of data streams, indices, and aliases used to limit the request.
+        Supports wildcards (`*`). To target all data streams and indices, omit this parameter or use `*` or `_all`.
+      required: true
+      schema:
+        $ref: '#/components/schemas/_common:Indices'
+      style: simple
+    cat.indices::query.format:
+      in: query
+      name: format
+      schema:
+        type: string
     indices.delete::path.index:
       in: path
       name: index

diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
@@ -22,6 +22,14 @@ chapters:
     request_body:
       payload:
         aliases: {}
+  - synopsis: This chapter should fail because the response content type does not match.
+    path: /_cat/indices/{index}
+    method: GET
+    parameters:
+      index: books
+      format: yaml
+    response:
+      status: 200
   - synopsis: This chapter should fail because the response data and schema are invalid.
     path: /{index}
     method: DELETE

diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml
@@ -14,6 +14,9 @@ chapters:
   - synopsis: This GET /_cat chapter returns text/plain and should pass.
     path: /_cat
     method: GET
+    response:
+      status: 200
+      content_type: text/plain
   - synopsis: This GET /_cat/health chapter returns application/json and should pass.
     path: /_cat/health
     parameters: