Remove ability to skip stories in tests.

CHANGELOG.md

@@ -36,6 +36,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Removed
 
+- Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358)
+
 ### Fixed
 
 - Fixed GitHub pages ([#215](https://github.com/opensearch-project/opensearch-api-specification/pull/215))

DEVELOPER_GUIDE.md

@@ -167,7 +167,6 @@ Below is the simplified version of the test story that tests the [index operatio
 ```yaml
 $schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
 
-skip: false # Skip this test story if set to true.
 description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
 
 prologues: [] # No prologues are needed for this story.

json_schemas/test_story.schema.yaml

@@ -4,10 +4,6 @@ type: object
 properties:
   $schema:
     type: string
-  skip:
-    type: boolean
-    description: If true, the story will be skipped.
-    default: false
   description:
     type: string
   prologues:

tests/_core/info.yaml

@@ -1,7 +1,6 @@
 
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test root endpoint.
 chapters:
   - synopsis: Get server info.

tests/indices/_doc.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test inserting and retrieving a doc.
 epilogues:
   - path: /movies

tests/indices/index.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: Test endpoints relevant the lifecycle of an index, from creation to deletion.
 epilogues:
   - path: /books

tests/ingest/pipeline.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: |
   Test the creation of an ingest pipeline with a text embedding processor.
 epilogues:

tests/ml/model_groups.yaml

@@ -1,6 +1,5 @@
 $schema: ../../json_schemas/test_story.schema.yaml
 
-skip: false
 description: |
   Test the creation of model groups.
 epilogues:

tools/src/tester/StoryEvaluator.ts

@@ -31,15 +31,6 @@ export default class StoryEvaluator {
   }
 
   async evaluate({ story, display_path, full_path }: StoryFile, dry_run: boolean = false): Promise<StoryEvaluation> {
-    if (story.skip) {
-      return {
-        result: Result.SKIPPED,
-        display_path,
-        full_path,
-        description: story.description,
-        chapters: []
-      }
-    }
     const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path)
     if (variables_error !== undefined) {
       return variables_error

tools/src/tester/types/story.types.ts

@@ -58,10 +58,6 @@ export type ReadChapter = Chapter & {
 
 export interface Story {
   $schema?: string;
-  /**
-   * If true, the story will be skipped.
-   */
-  skip?: boolean;
   description: string;
   prologues?: SupplementalChapter[];
   epilogues?: SupplementalChapter[];

tools/tests/tester/fixtures/empty_with_all_the_parts.yaml

@@ -1,7 +1,5 @@
 $schema: ../json_schemas/test_story.schema.yaml
 
-skip: false
-
 description: A story with all its parts.
 
 prologues:

tools/tests/tester/fixtures/fail_non_existent_variable.yaml

@@ -1,6 +1,5 @@
 $schema: ../json_schemas/test_story.schema.yaml
 
-skip: false
 description: |
   This test story checks that we can create a model group
 chapters:

tools/tests/tester/integ/StoryEvaluator.test.ts

@@ -17,12 +17,6 @@ test('passed', async () => {
   expect(actual).toEqual(expected)
 })
 
-test('skipped', async () => {
-  const actual = await load_actual_evaluation(story_evaluator, 'skipped')
-  const expected = load_expected_evaluation('skipped')
-  expect(actual).toEqual(expected)
-})
-
 test('failed/not_found', async () => {
   const actual = await load_actual_evaluation(story_evaluator, 'failed/not_found')
   const expected = load_expected_evaluation('failed/not_found')

tools/tests/tester/integ/TestRunner.test.ts

@@ -24,13 +24,12 @@ test('stories folder', async () => {
     actual_evaluations.push(rest)
   }
 
-  const skipped = load_expected_evaluation('skipped', true)
   const passed = load_expected_evaluation('passed', true)
   const not_found = load_expected_evaluation('failed/not_found', true)
   const invalid_data = load_expected_evaluation('failed/invalid_data', true)
   const chapter_error = load_expected_evaluation('error/chapter_error', true)
   const prologue_error = load_expected_evaluation('error/prologue_error', true)
 
-  const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
+  const expected_evaluations = [passed, chapter_error, prologue_error, invalid_data, not_found]
   expect(actual_evaluations).toEqual(expected_evaluations)
 })