Add support for pending tests

CHANGELOG.md

@@ -43,6 +43,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added `POST /_plugins/_ml/_train/{algorithm_name}`, `_predict/{algorithm_name}/{model_id}`, and `_train_predict/{algorithm_name}` ([#755](https://github.com/opensearch-project/opensearch-api-specification/pull/755))
 - Added `PUT /_plugins/_ml/model_groups/{model_group_id}`, `GET /_plugins/_ml/model_groups/_search`, and `POST /_plugins/_ml/model_groups/_search` ([#760](https://github.com/opensearch-project/opensearch-api-specification/pull/760))
 - Added `GET /_plugins/_ml/connectors/{connector_id}`, `_search`, `POST /_plugins/_ml/connectors/_search`, and `PUT /_plugins/_ml/connectors/{connector_id}` ([#764](https://github.com/opensearch-project/opensearch-api-specification/pull/764))
+- Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))

TESTING_GUIDE.md

@@ -193,6 +193,11 @@ chapters:
   - synopsis: Delete the `books` index.
     path: /{index}
     method: DELETE
+    pending: | # This test is included but marked as pending, meaning it will not be executed. An explanation is provided to clarify why it has been skipped.
+       SSL needs to be configured to trust itself. 
+       For this to work, will need to:
+       1. set reindex.remote.allowlist=localhost:9200 in docker-compose.yml
+       2. setup SSL in a way that the docker instance trusts its own cert
     parameters:
       index: books
 ```

json_schemas/test_story.schema.yaml

@@ -44,6 +44,9 @@ definitions:
             type: string
             description: A brief description of the chapter.
             pattern: ^\p{Lu}[\s\S]*\.$|^\p{Lu}[\s\S]*\. \[(GET|PUT|POST|DELETE|PATCH|HEAD|OPTIONS)\]$
+          pending:
+            type: string
+            description: An explanation is provided to clarify why it has been skipped.
           response:
             $ref: '#/definitions/ExpectedResponse'
           warnings:

tools/src/tester/StoryEvaluator.ts

@@ -111,6 +111,7 @@ export default class StoryEvaluator {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
+
       if (dry_run) {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) {
@@ -119,6 +120,9 @@ export default class StoryEvaluator {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } })
       } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } })
+      } else if (chapter.pending != null && chapter.pending) {
+        evaluations.push({ title, overall: { result: Result.IGNORED, message:  chapter.pending } })
+        continue
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
         has_errors = has_errors || evaluation.overall.result === Result.ERROR

tools/src/tester/types/eval.types.ts

@@ -43,6 +43,7 @@ export interface ChapterEvaluation {
   overall: Evaluation,
   operation?: Operation,
   path?: string,
+  pending?: string,
   request?: {
     parameters?: Record<string, Evaluation>
     request?: Evaluation
@@ -79,6 +80,7 @@ export type EvaluationWithOutput = {
 
 export enum Result {
   PASSED = 'PASSED',
+  IGNORED = 'IGNORED',
   FAILED = 'FAILED',
   SKIPPED = 'SKIPPED',
   ERROR = 'ERROR',

tools/src/tester/types/story.types.ts

@@ -90,6 +90,7 @@ export type Chapter = ChapterRequest & {
    * A brief description of the chapter.
    */
   synopsis: string;
+  pending?: string;
   response?: ExpectedResponse;
   warnings?: Warnings;
 };

tools/tests/tester/fixtures/evals/ignored/ignored.yaml

@@ -0,0 +1,15 @@
+display_path: ignored/ignored.yaml
+full_path: tools/tests/tester/fixtures/stories/ignored/ignored.yaml
+
+result: PASSED
+description: This story should ignored.
+prologues: []
+chapters:
+  - title: This PUT /{index} chapter should pending.
+    overall:
+      result: IGNORED
+      message: This chapter is ignored because it is not relevant to the current test.
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED

tools/tests/tester/fixtures/stories/ignored/ignored.yaml

@@ -0,0 +1,14 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should ignored.
+epilogues:
+  - path: /books
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: This PUT /{index} chapter should pending.
+    path: /{index}
+    pending: This chapter is ignored because it is not relevant to the current test.
+    method: PUT
+    parameters:
+      index: books

tools/tests/tester/integ/StoryEvaluator.test.ts

@@ -69,6 +69,12 @@ test('skipped/semver', async () => {
   expect(actual).toEqual(expected)
 })
 
+test('ignored', async () => {
+  const actual = await load_actual_evaluation(story_evaluator, 'ignored/ignored')
+  const expected = load_expected_evaluation('ignored/ignored')
+  expect(actual).toEqual(expected)
+})
+
 test('with an unexpected error deserializing data', async () => {
   opensearch_http_client.request = jest.fn().mockRejectedValue(new Error('This was unexpected.'))
   const actual = await load_actual_evaluation(story_evaluator, 'passed/passed')

tools/tests/tester/integ/TestRunner.test.ts

@@ -34,6 +34,7 @@ test('stories folder', async () => {
     'error/prologue_error',
     'failed/invalid_data',
     'failed/not_found',
+    'ignored/ignored',
     'passed/multiple_methods',
     'passed/passed',
     'passed/value_type',