Added support for multiple test verbs.

CHANGELOG.md

@@ -32,6 +32,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added response schema for `DELETE /_plugins/_rollup/jobs/{id}`, `POST /_plugins/_rollup/jobs/{id}/_start` and `_stop` ([#716](https://github.com/opensearch-project/opensearch-api-specification/pull/716))
 - Added response schema for `PUT` and `DELETE /_plugins/_transform/{id}` ([#722](https://github.com/opensearch-project/opensearch-api-specification/pull/716))
 - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717))
+- Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
@@ -64,6 +65,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed content-type of `GET /_plugins/_observability/_local/stats` ([#711](https://github.com/opensearch-project/opensearch-api-specification/pull/711))
 - Fixed `tenant` in `ObservabilityObject` request body to not be required ([#711](https://github.com/opensearch-project/opensearch-api-specification/pull/711))
 - Fixed response code in `PUT /_plugins/_rollup/jobs/{id}` ([#716](https://github.com/opensearch-project/opensearch-api-specification/pull/716))
+- Fixed response schema for `/_render/template` and `/_render/template/{id}` ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724))
 
 ### Changed
 - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))

TESTING_GUIDE.md

@@ -12,6 +12,7 @@
       - [FAILED  Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics)
   - [Writing Spec Tests](#writing-spec-tests)
     - [Simple Test Story](#simple-test-story)
+    - [Testing Multiple Verbs](#testing-multiple-verbs)
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
     - [Managing Versions](#managing-versions)
     - [Managing Distributions](#managing-distributions)
@@ -182,6 +183,18 @@ chapters:
       index: books
 ```
 
+### Testing Multiple Verbs
+
+Some APIs allow multiple verbs for the same effect. Specify multiple verbs as follows and the test tool will execute both.
+
+```yaml
+- synopsis: Use POST and PUT interchangeably.
+  path: /{index}
+  method:
+    - POST
+    - PUT
+```
+
 ### Using Output from Previous Chapters
 
 Consider the following chapters in [ml/model_groups](tests/plugins/ml/ml/model_groups.yaml) test story:

json_schemas/test_story.schema.yaml

@@ -67,6 +67,11 @@ definitions:
             items:
               type: integer
 
+  HttpMethod:
+    type: string
+    # eslint-disable-next-line yml/sort-sequence-values
+    enum: [GET, PUT, POST, DELETE, PATCH, HEAD, OPTIONS]
+
   ChapterRequest:
     type: object
     properties:
@@ -76,9 +81,11 @@ definitions:
       path:
         type: string
       method:
-        type: string
-        # eslint-disable-next-line yml/sort-sequence-values
-        enum: [GET, PUT, POST, DELETE, PATCH, HEAD, OPTIONS]
+        oneOf:
+          - type: array
+            items:
+              $ref: '#/definitions/HttpMethod'
+          - $ref: '#/definitions/HttpMethod'
       parameters:
         type: object
         additionalProperties:

spec/namespaces/_core.yaml

@@ -2482,16 +2482,15 @@ components:
           schema:
             type: object
             properties:
-              file:
-                type: string
+              id:
+                $ref: '../schemas/_common.yaml#/components/schemas/Id'
               params:
                 description: |-
                   Key-value pairs used to replace Mustache variables in the template.
                   The key is the variable name.
                   The value is the variable value.
                 type: object
-                additionalProperties:
-                  type: object
+                additionalProperties: true
               source:
                 description: |-
                   An inline search template.

tests/default/_core/render/template.yaml

@@ -0,0 +1,62 @@
+$schema: ../../../../json_schemas/test_story.schema.yaml
+
+description: Test rendering a search template as a search query.
+
+prologues:
+  - path: /_scripts/movie_template
+    method: POST
+    request:
+      content_type: application/json
+      payload:
+        script:
+          lang: mustache
+          source: >
+            {
+              "query": {
+                "match": {
+                  "{{field}}": "{{value}}"
+                }
+              }
+            }
+epilogues:
+  - path: /_scripts/movie_template
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: Render the movie template (request payload).
+    path: /_render/template
+    method:
+      - GET
+      - POST
+    request:
+      payload:
+        id: movie_template
+        params:
+          field: director
+          value: Quentin Tarantino
+    response:
+      status: 200
+      payload:
+        template_output:
+          query:
+            match:
+              director: Quentin Tarantino
+  - synopsis: Render the movie template (path).
+    path: /_render/template/{id}
+    method:
+      - GET
+      - POST
+    parameters:
+      id: movie_template
+    request:
+      payload:
+        params:
+          field: director
+          value: Christopher Nolan
+    response:
+      status: 200
+      payload:
+        template_output:
+          query:
+            match:
+              director: Christopher Nolan

tests/default/search/template.yaml

@@ -5,6 +5,9 @@ epilogues:
   - path: /movies
     method: DELETE
     status: [200, 404]
+  - path: /movies
+    method: DELETE
+    status: [200, 404]
 prologues:
   - path: /_bulk
     method: POST

tools/src/tester/ChapterEvaluator.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type Chapter, type ActualResponse, type Payload } from './types/story.types'
+import { type ActualResponse, type Payload } from './types/story.types'
 import { type ChapterEvaluation, type Evaluation, Result, EvaluationWithOutput } from './types/eval.types'
 import { type ParsedOperation } from './types/spec.types'
 import { overall_result } from './helpers'
@@ -21,6 +21,7 @@ import _ from 'lodash'
 import { Logger } from 'Logger'
 import { sleep, to_json } from '../helpers'
 import { APPLICATION_JSON } from "./MimeTypes";
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class ChapterEvaluator {
   private readonly logger: Logger
@@ -35,7 +36,7 @@ export default class ChapterEvaluator {
     this.logger = logger
   }
 
-  async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
+  async evaluate(chapter: ParsedChapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
     if (skip) return { title: chapter.synopsis, overall: { result: Result.SKIPPED } }
 
     const operation = this._operation_locator.locate_operation(chapter)
@@ -61,7 +62,7 @@ export default class ChapterEvaluator {
     return result
   }
 
-  async #evaluate(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
+  async #evaluate(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const params = this.#evaluate_parameters(chapter, operation, story_outputs)
     const request = this.#evaluate_request(chapter, operation, story_outputs)
@@ -110,7 +111,7 @@ export default class ChapterEvaluator {
     return result
   }
 
-  #evaluate_parameters(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs): Record<string, Evaluation> {
+  #evaluate_parameters(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs): Record<string, Evaluation> {
     const parameters: Record<string, any> = story_outputs.resolve_value(chapter.parameters) ?? {}
     return Object.fromEntries(Object.entries(parameters).map(([name, parameter]) => {
       const schema = operation.parameters[name]?.schema
@@ -120,7 +121,7 @@ export default class ChapterEvaluator {
     }))
   }
 
-  #evaluate_request(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs): Evaluation {
+  #evaluate_request(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs): Evaluation {
     if (chapter.request?.payload === undefined) return { result: Result.PASSED }
     const content_type = chapter.request.content_type ?? APPLICATION_JSON
     const schema = operation.requestBody?.content[content_type]?.schema
@@ -129,7 +130,7 @@ export default class ChapterEvaluator {
     return this._schema_validator.validate(schema, payload)
   }
 
-  #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation {
+  #evaluate_status(chapter: ParsedChapter, response: ActualResponse): Evaluation {
     const expected_status = chapter.response?.status ?? 200
     if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED }
 
@@ -166,7 +167,7 @@ export default class ChapterEvaluator {
     return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
   }
 
-  #evaluate_payload_schema(chapter: Chapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
+  #evaluate_payload_schema(chapter: ParsedChapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
     const content_type = chapter.response?.content_type ?? APPLICATION_JSON
 
     if (response.content_type !== content_type) {

tools/src/tester/ChapterReader.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types'
+import { type ActualResponse, type Parameter } from './types/story.types'
 import { type OpenSearchHttpClient } from '../OpenSearchHttpClient'
 import { type StoryOutputs } from './StoryOutputs'
 import { Logger } from 'Logger'
@@ -18,6 +18,7 @@ import CBOR from 'cbor'
 import SMILE from 'smile-js'
 import { APPLICATION_CBOR, APPLICATION_JSON, APPLICATION_SMILE, APPLICATION_YAML, TEXT_PLAIN } from "./MimeTypes";
 import _ from 'lodash'
+import { ParsedChapterRequest } from './types/parsed_story.types'
 
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
@@ -28,7 +29,7 @@ export default class ChapterReader {
     this.logger = logger
   }
 
-  async read (chapter: ChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
+  async read (chapter: ParsedChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
     const response: Record<string, any> = {}
     const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
     const [url_path, params] = this.#parse_url(chapter.path, resolved_params)

tools/src/tester/OperationLocator.ts

@@ -9,9 +9,9 @@
 
 import { type OpenAPIV3 } from 'openapi-types'
 import { resolve_ref } from '../helpers'
-import { type Chapter } from './types/story.types'
 import { type ParsedOperation } from './types/spec.types'
 import _ from 'lodash'
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class OperationLocator {
   private readonly spec: OpenAPIV3.Document
@@ -21,7 +21,7 @@ export default class OperationLocator {
     this.spec = spec
   }
 
-  locate_operation (chapter: Chapter): ParsedOperation | undefined {
+  locate_operation (chapter: ParsedChapter): ParsedOperation | undefined {
     const path = chapter.path
     const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
     const cache_key = path + method

tools/src/tester/StoryEvaluator.ts

@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { ChapterRequest, Parameter, type Chapter, type Story, type SupplementalChapter } from './types/story.types'
+import { Parameter } from './types/story.types'
 import { type StoryFile, type ChapterEvaluation, Result, type StoryEvaluation, OutputReference } from './types/eval.types'
 import type ChapterEvaluator from './ChapterEvaluator'
 import { overall_result } from './helpers'
@@ -16,6 +16,7 @@ import SupplementalChapterEvaluator from './SupplementalChapterEvaluator'
 import { ChapterOutput } from './ChapterOutput'
 import * as semver from '../_utils/semver'
 import _ from 'lodash'
+import { ParsedChapter, ParsedChapterRequest, ParsedStory, ParsedSupplementalChapter } from './types/parsed_story.types'
 
 export default class StoryEvaluator {
   private readonly _chapter_evaluator: ChapterEvaluator
@@ -85,14 +86,14 @@ export default class StoryEvaluator {
     return result
   }
 
-  #chapter_warnings(story: Story): string[] | undefined {
+  #chapter_warnings(story: ParsedStory): string[] | undefined {
     const result = _.compact([
       this.#warning_if_mismatched_chapter_paths(story)
     ])
     return result.length > 0 ? result : undefined
   }
 
-  #warning_if_mismatched_chapter_paths(story: Story): string | undefined {
+  #warning_if_mismatched_chapter_paths(story: ParsedStory): string | undefined {
     if (story.warnings?.['multiple-paths-detected'] === false) return
     const paths = _.compact(_.map(story.chapters, (chapter) => {
       if (chapter.warnings?.['multiple-paths-detected'] === false) return
@@ -105,7 +106,7 @@ export default class StoryEvaluator {
     }
   }
 
-  async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
+  async #evaluate_chapters(chapters: ParsedChapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       if (dry_run) {
@@ -132,7 +133,7 @@ export default class StoryEvaluator {
     return evaluations
   }
 
-  async #evaluate_supplemental_chapters(chapters: SupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
+  async #evaluate_supplemental_chapters(chapters: ParsedSupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
     let has_errors = false
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
@@ -152,7 +153,7 @@ export default class StoryEvaluator {
   }
 
   // TODO: Refactor and move this logic into StoryValidator
-  static check_story_variables(story: Story, display_path: string, full_path: string): StoryEvaluation | undefined {
+  static check_story_variables(story: ParsedStory, display_path: string, full_path: string): StoryEvaluation | undefined {
     const story_outputs = new StoryOutputs()
     const prologues = (story.prologues ?? []).map((prologue) => {
       return StoryEvaluator.#check_chapter_variables(prologue, story_outputs)
@@ -178,7 +179,7 @@ export default class StoryEvaluator {
     }
   }
 
-  static #check_chapter_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
+  static #check_chapter_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
     const title = `${chapter.method} ${chapter.path}`
     const error = StoryEvaluator.#check_used_variables(chapter, story_outputs)
     if (error !== undefined) {
@@ -201,7 +202,7 @@ export default class StoryEvaluator {
    * @param story_outputs
    * @returns
    */
-  static #check_used_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
+  static #check_used_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
     const variables = new Set<OutputReference>()
     const title = `${chapter.method} ${chapter.path}`
     StoryEvaluator.#extract_params_variables(chapter.parameters ?? {}, variables)

tools/src/tester/StoryParser.ts

@@ -0,0 +1,39 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import _ from "lodash";
+import { ChapterRequest, Story } from "./types/story.types";
+import { ParsedChapter, ParsedChapterRequest, ParsedStory } from "./types/parsed_story.types";
+
+export default class StoryParser {
+  static parse(story: Story): ParsedStory {
+    return {
+      ...story,
+      chapters: this.#expand_chapters(story.chapters) as ParsedChapter[],
+      prologues: this.#expand_chapters(story.prologues),
+      epilogues: this.#expand_chapters(story.epilogues)
+    }
+  }
+
+  static #chapter_methods(methods: string[] | string): string[] {
+    return [...(Array.isArray(methods) ? methods : [methods])]
+  }
+
+  static #expand_chapters(chapters?: ChapterRequest[]): ParsedChapterRequest[] {
+    if (chapters === undefined) return []
+    return  _.flatMap(_.map(chapters, (chapter) => {
+      return _.map(this.#chapter_methods(chapter.method), (method) => {
+        return {
+          ...chapter,
+          method
+        }
+      })
+    })) as ParsedChapterRequest[]
+  }
+}