# additional_success_status

Signed-off-by: Theo Truong <[email protected]>

json_schemas/test_story.schema.yaml

@@ -55,10 +55,12 @@ definitions:
       - $ref: '#/definitions/ChapterRequest'
       - type: object
         properties:
-          ignore_errors:
-            description: If true, treat all non-2XX responses as successful.
-            type: boolean
-            default: false
+          additional_success_status:
+            description: Additional HTTP status codes, other than 2XX codes, that are considered successful.
+            type: array
+            items:
+              type: integer
+        additionalProperties: false
 
   ChapterRequest:
     type: object

tests/index_lifecycle.yaml

@@ -3,9 +3,12 @@ $schema: ../../json_schemas/test_story.schema.yaml
 skip: false
 description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
 epilogues:
-  - path: /books,movies,games
+  - path: /books
     method: DELETE
-    ignore_errors: false
+    additional_success_status: [404]
+  - path: /games
+    method: DELETE
+    additional_success_status: [404]
 chapters:
   - synopsis: Create an index named `books` with mappings and settings.
     path: /{index}
@@ -31,16 +34,12 @@ chapters:
     method: PUT
     parameters:
       index: games
-    response:
-      status: 200
 
   - synopsis: Check if the index `books` exists. It should.
     path: /{index}
     method: HEAD
     parameters:
       index: books
-    response:
-      status: 200
 
   - synopsis: Check if the index `movies` exists. It should not.
     path: /{index}
@@ -56,29 +55,21 @@ chapters:
     parameters:
       index: books,games
       flat_settings: true
-    response:
-      status: 200
 
   - synopsis: Close the `books` index.
     path: /{index}/_close
     method: POST
     parameters:
       index: books
-    response:
-      status: 200
 
   - synopsis: Open the `books` index.
     path: /{index}/_open
     method: POST
     parameters:
       index: books
-    response:
-      status: 200
 
   - synopsis: Delete the `books` and `games` indices.
     path: /{index}
     method: DELETE
     parameters:
       index: books,games
-    response:
-      status: 200

tools/src/tester/ChapterEvaluator.ts

@@ -3,14 +3,13 @@ import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.ty
 import { type ParsedOperation } from './types/spec.types'
 import { overall_result } from './helpers'
 import type ChapterReader from './ChapterReader'
-import { ServerError } from './ChapterReader'
 import SharedResources from './SharedResources'
 import type SpecParser from './SpecParser'
 import type SchemaValidator from './SchemaValidator'
 
 export default class ChapterEvaluator {
   chapter: Chapter
-  skip_payload: boolean = false
+  skip_payload_evaluation: boolean = false
   spec_parser: SpecParser
   chapter_reader: ChapterReader
   schema_validator: SchemaValidator
@@ -23,23 +22,18 @@ export default class ChapterEvaluator {
   }
 
   async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
-    try {
-      if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
-      const operation = this.spec_parser.locate_operation(this.chapter)
-      const response = await this.chapter_reader.read(this.chapter, true)
-      const params = this.#evaluate_parameters(operation)
-      const request_body = this.#evaluate_request_body(operation)
-      const status = this.#evaluate_status(response)
-      const payload = this.#evaluate_payload(operation, response)
-      return {
-        title: this.chapter.synopsis,
-        overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
-        request: { parameters: params, requestBody: request_body },
-        response: { status, payload }
-      }
-    } catch (error) {
-      if (!(error instanceof ServerError)) throw error
-      return { title: this.chapter.synopsis, overall: { result: Result.ERROR, message: error.message, error: error.original_error } }
+    if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
+    const operation = this.spec_parser.locate_operation(this.chapter)
+    const response = await this.chapter_reader.read(this.chapter)
+    const params = this.#evaluate_parameters(operation)
+    const request_body = this.#evaluate_request_body(operation)
+    const status = this.#evaluate_status(response)
+    const payload = this.#evaluate_payload(operation, response)
+    return {
+      title: this.chapter.synopsis,
+      overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
+      request: { parameters: params, requestBody: request_body },
+      response: { status, payload }
     }
   }
 
@@ -63,12 +57,16 @@ export default class ChapterEvaluator {
   #evaluate_status (response: ActualResponse): Evaluation {
     const expected_status = this.chapter.response?.status ?? 200
     if (response.status === expected_status) return { result: Result.PASSED }
-    this.skip_payload = true
-    return { result: Result.FAILED, message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.` }
+    this.skip_payload_evaluation = true
+    return {
+      result: Result.ERROR,
+      message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}. ${response.message}`,
+      error: response.error as Error
+    }
   }
 
   #evaluate_payload (operation: ParsedOperation, response: ActualResponse): Evaluation {
-    if (this.skip_payload) return { result: Result.SKIPPED }
+    if (this.skip_payload_evaluation) return { result: Result.SKIPPED }
     const content_type = response.content_type ?? 'application/json'
     const content = operation.responses[response.status]?.content[content_type]
     const schema = content?.schema

tools/src/tester/ChapterReader.ts

@@ -1,14 +1,6 @@
 import axios from 'axios'
 import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types'
 
-export class ServerError extends Error {
-  original_error: Error
-  constructor (message: string, error: Error) {
-    super(message)
-    this.original_error = error
-  }
-}
-
 // A lightweight client for testing the API
 export default class ChapterReader {
   url: string
@@ -17,7 +9,7 @@ export default class ChapterReader {
     this.url = url
   }
 
-  async read (chapter: ChapterRequest, ignore_errors: boolean = false): Promise<ActualResponse> {
+  async read (chapter: ChapterRequest): Promise<ActualResponse> {
     const response: Record<string, any> = {}
     const [url, params] = this.#parse_url(chapter.path, chapter.parameters ?? {})
     await axios.request({
@@ -31,11 +23,10 @@ export default class ChapterReader {
       response.payload = r.data
     }).catch(e => {
       if (e.response == null) throw e
-      if (!ignore_errors) throw new ServerError(e.response.data.error.reason as string, e as Error)
+      response.status = e.response.status
       response.content_type = e.response.headers['content-type'].split(';')[0]
       response.payload = e.response.data?.error
       response.message = e.response.data?.error?.reason
-      response.status = e.response.status
       response.error = e
     })
     return response as ActualResponse

tools/src/tester/ResultsDisplayer.ts

@@ -16,24 +16,21 @@ function cyan (text: string): string { return `\x1b[36m${text}\x1b[0m` }
 function gray (text: string): string { return `\x1b[90m${text}\x1b[0m` }
 function magenta (text: string): string { return `\x1b[35m${text}\x1b[0m` }
 
-interface ResultDisplayerOptions {
-  gap?: number
+export interface DisplayOptions {
+  tab_size?: number
   verbose?: boolean
-  ignored_results: Set<Result>
 }
 
 export default class ResultsDisplayer {
   evaluation: StoryEvaluation
   skip_components: boolean
-  gap: number
-  ignored_results: Set<Result>
+  tab_size: number
   verbose: boolean
 
-  constructor (evaluation: StoryEvaluation, opts: ResultDisplayerOptions) {
+  constructor (evaluation: StoryEvaluation, opts: DisplayOptions) {
     this.evaluation = evaluation
     this.skip_components = [Result.PASSED, Result.SKIPPED].includes(evaluation.result)
-    this.ignored_results = opts.ignored_results
-    this.gap = opts.gap ?? 4
+    this.tab_size = opts.tab_size ?? 4
     this.verbose = opts.verbose ?? false
   }
 
@@ -55,13 +52,13 @@ export default class ResultsDisplayer {
   #display_chapters (evaluations: ChapterEvaluation[], title: string): void {
     if (this.skip_components || evaluations.length === 0) return
     const result = overall_result(evaluations.map(e => e.overall))
-    this.#display_evaluation({ result }, title, this.gap)
+    this.#display_evaluation({ result }, title, this.tab_size)
     if (result === Result.PASSED) return
     for (const evaluation of evaluations) this.#display_chapter(evaluation)
   }
 
   #display_chapter (chapter: ChapterEvaluation): void {
-    this.#display_evaluation(chapter.overall, i(chapter.title), this.gap * 2)
+    this.#display_evaluation(chapter.overall, i(chapter.title), this.tab_size * 2)
     if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return
 
     this.#display_parameters(chapter.request?.parameters ?? {})
@@ -73,31 +70,29 @@ export default class ResultsDisplayer {
   #display_parameters (parameters: Record<string, Evaluation>): void {
     if (Object.keys(parameters).length === 0) return
     const result = overall_result(Object.values(parameters))
-    this.#display_evaluation({ result }, 'PARAMETERS', this.gap * 3)
+    this.#display_evaluation({ result }, 'PARAMETERS', this.tab_size * 3)
     if (result === Result.PASSED) return
     for (const [name, evaluation] of Object.entries(parameters)) {
-      this.#display_evaluation(evaluation, name, this.gap * 4)
+      this.#display_evaluation(evaluation, name, this.tab_size * 4)
     }
   }
 
   #display_request_body (evaluation: Evaluation | undefined): void {
     if (evaluation == null) return
-    this.#display_evaluation(evaluation, 'REQUEST BODY', this.gap * 3)
+    this.#display_evaluation(evaluation, 'REQUEST BODY', this.tab_size * 3)
   }
 
   #display_status (evaluation: Evaluation | undefined): void {
     if (evaluation == null) return
-    this.#display_evaluation(evaluation, 'RESPONSE STATUS', this.gap * 3)
+    this.#display_evaluation(evaluation, 'RESPONSE STATUS', this.tab_size * 3)
   }
 
   #display_payload (evaluation: Evaluation | undefined): void {
     if (evaluation == null) return
-    this.#display_evaluation(evaluation, 'RESPONSE PAYLOAD', this.gap * 3)
+    this.#display_evaluation(evaluation, 'RESPONSE PAYLOAD', this.tab_size * 3)
   }
 
   #display_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void {
-    if (evaluation.result === Result.PASSED && this.ignored_results.has(Result.PASSED)) return
-    if (evaluation.result === Result.SKIPPED && this.ignored_results.has(Result.SKIPPED)) return
     const result = padding(this.#result(evaluation.result), 0, prefix)
     const message = evaluation.message != null ? `${gray('(' + evaluation.message + ')')}` : ''
     console.log(`${result} ${title} ${message}`)

tools/src/tester/StoryEvaluator.ts

@@ -2,7 +2,6 @@ import { type Chapter, type Story, type SupplementalChapter } from './types/stor
 import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/eval.types'
 import ChapterEvaluator from './ChapterEvaluator'
 import type ChapterReader from './ChapterReader'
-import { ServerError } from './ChapterReader'
 import SharedResources from './SharedResources'
 
 export interface StoryFile {
@@ -69,14 +68,14 @@ export default class StoryEvaluator {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       const title = `${chapter.method} ${chapter.path}`
-      try {
-        await this.chapter_reader.read(chapter, chapter.ignore_errors)
+      const response = await this.chapter_reader.read(chapter)
+      const additional_success_status = chapter.additional_success_status ?? []
+      if (!response.error || additional_success_status.includes(response.status)) {
         evaluations.push({ title, overall: { result: Result.PASSED } })
-      } catch (error) {
-        if (!(error instanceof ServerError)) throw error
+      } else {
         this.result = Result.ERROR
         this.has_errors = true
-        evaluations.push({ title, overall: { result: Result.ERROR, message: (error).message, error: error.original_error } })
+        evaluations.push({ title, overall: { result: Result.ERROR, message: response.message, error: response.error as Error } })
       }
     }
     return evaluations

tools/src/tester/TestsRunner.ts

@@ -6,15 +6,12 @@ import StoryEvaluator, { type StoryFile } from './StoryEvaluator'
 import fs from 'fs'
 import { type Story } from './types/story.types'
 import { read_yaml } from '../../helpers'
-import { type Result, type StoryEvaluation } from './types/eval.types'
-import ResultsDisplayer from './ResultsDisplayer'
+import { type StoryEvaluation } from './types/eval.types'
+import ResultsDisplayer, { type DisplayOptions } from './ResultsDisplayer'
 import SharedResources from './SharedResources'
 import { resolve, basename } from 'path'
 
-interface TestsRunnerOptions {
-  ignored_results: Set<Result> // Chapter results to ignore when displaying results
-  verbose: boolean // Whether to print the full stack trace of errors
-}
+type TestsRunnerOptions = DisplayOptions & Record<string, any>
 
 export default class TestsRunner {
   path: string // Path to a story file or a directory containing story files

tools/src/tester/start.ts

@@ -2,20 +2,21 @@ import OpenApiMerger from '../merger/OpenApiMerger'
 import { LogLevel } from '../Logger'
 import TestsRunner from './TestsRunner'
 import { Command, Option } from '@commander-js/extra-typings'
-import { Result } from './types/eval.types'
 
 const command = new Command()
   .description('Run test stories against the OpenSearch spec.')
   .addOption(new Option('--spec, --spec_path <path>', 'path to the root folder of the multi-file spec').default('./spec'))
   .addOption(new Option('--tests, --tests_path <path>', 'path to the root folder of the tests').default('./tests'))
-  .addOption(new Option('--ignore <results>', 'comma-separated list of chapter results to ignore. Ignorable results are SKIPPED and PASSED').default('PASSED'))
+  .addOption(new Option('--tab_size <size>', 'tab size for displayed results').default('4'))
   .addOption(new Option('--verbose', 'whether to print the full stack trace of errors'))
   .allowExcessArguments(false)
   .parse()
 
 const opts = command.opts()
-const ignored_results = new Set(opts.ignore.split(',').map(r => Result[r as keyof typeof Result]))
-const verbose = opts.verbose ?? false
+const display_options = {
+  verbose: opts.verbose ?? false,
+  tab_size: Number.parseInt(opts.tab_size)
+}
 const spec = (new OpenApiMerger(opts.spec_path, LogLevel.error)).merge()
-const runner = new TestsRunner(spec, opts.tests_path, { ignored_results, verbose })
+const runner = new TestsRunner(spec, opts.tests_path, display_options)
 void runner.run().then(() => { console.log('Tests run successfully.') })

tools/src/tester/types/story.types.ts

@@ -13,9 +13,9 @@
  */
 export type SupplementalChapter = ChapterRequest & {
   /**
-   * If true, treat all non-2XX responses as successful.
+   * Additional HTTP status codes, other than 2XX codes, that are considered successful.
    */
-  ignore_errors?: boolean;
+  additional_success_status?: number[];
 };
 /**
  * This interface was referenced by `Story`'s JSON-Schema