diff --git a/CHANGELOG.md b/CHANGELOG.md index 53beddde3..7885b6a55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Removed +- Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358) + ### Fixed - Fixed GitHub pages ([#215](https://github.com/opensearch-project/opensearch-api-specification/pull/215)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 676bf383c..64f4f8358 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -167,7 +167,6 @@ Below is the simplified version of the test story that tests the [index operatio ```yaml $schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly. -skip: false # Skip this test story if set to true. description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion. prologues: [] # No prologues are needed for this story. diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 126426ff0..d0716669f 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -4,10 +4,6 @@ type: object properties: $schema: type: string - skip: - type: boolean - description: If true, the story will be skipped. - default: false description: type: string prologues: diff --git a/tests/_core/info.yaml b/tests/_core/info.yaml index d46ca3eae..6635498bc 100644 --- a/tests/_core/info.yaml +++ b/tests/_core/info.yaml @@ -1,7 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -skip: false description: Test root endpoint. chapters: - synopsis: Get server info. diff --git a/tests/indices/_doc.yaml b/tests/indices/_doc.yaml index b4789f0c7..cf01403b8 100644 --- a/tests/indices/_doc.yaml +++ b/tests/indices/_doc.yaml @@ -1,6 +1,5 @@ $schema: ../../json_schemas/test_story.schema.yaml -skip: false description: Test inserting and retrieving a doc. epilogues: - path: /movies diff --git a/tests/indices/index.yaml b/tests/indices/index.yaml index 3873b0230..456078f4a 100644 --- a/tests/indices/index.yaml +++ b/tests/indices/index.yaml @@ -1,6 +1,5 @@ $schema: ../../json_schemas/test_story.schema.yaml -skip: false description: Test endpoints relevant the lifecycle of an index, from creation to deletion. epilogues: - path: /books diff --git a/tests/ingest/pipeline.yaml b/tests/ingest/pipeline.yaml index 2408d79d3..48144a9a3 100644 --- a/tests/ingest/pipeline.yaml +++ b/tests/ingest/pipeline.yaml @@ -1,6 +1,5 @@ $schema: ../../json_schemas/test_story.schema.yaml -skip: false description: | Test the creation of an ingest pipeline with a text embedding processor. epilogues: diff --git a/tests/ml/model_groups.yaml b/tests/ml/model_groups.yaml index 4fdb37000..71b794c2a 100644 --- a/tests/ml/model_groups.yaml +++ b/tests/ml/model_groups.yaml @@ -1,6 +1,5 @@ $schema: ../../json_schemas/test_story.schema.yaml -skip: false description: | Test the creation of model groups. epilogues: diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 231fe7d03..3e602a1c1 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -31,15 +31,6 @@ export default class StoryEvaluator { } async evaluate({ story, display_path, full_path }: StoryFile, dry_run: boolean = false): Promise { - if (story.skip) { - return { - result: Result.SKIPPED, - display_path, - full_path, - description: story.description, - chapters: [] - } - } const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 38ead0bc3..257eb3ca8 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -58,10 +58,6 @@ export type ReadChapter = Chapter & { export interface Story { $schema?: string; - /** - * If true, the story will be skipped. - */ - skip?: boolean; description: string; prologues?: SupplementalChapter[]; epilogues?: SupplementalChapter[]; diff --git a/tools/tests/tester/fixtures/empty_with_all_the_parts.yaml b/tools/tests/tester/fixtures/empty_with_all_the_parts.yaml index 9ea08619d..f1eeaeb79 100644 --- a/tools/tests/tester/fixtures/empty_with_all_the_parts.yaml +++ b/tools/tests/tester/fixtures/empty_with_all_the_parts.yaml @@ -1,7 +1,5 @@ $schema: ../json_schemas/test_story.schema.yaml -skip: false - description: A story with all its parts. prologues: diff --git a/tools/tests/tester/fixtures/evals/skipped.yaml b/tools/tests/tester/fixtures/evals/skipped.yaml deleted file mode 100644 index 663cb1acf..000000000 --- a/tools/tests/tester/fixtures/evals/skipped.yaml +++ /dev/null @@ -1,7 +0,0 @@ -display_path: skipped.yaml -full_path: "tools/tests/tester/fixtures/stories/skipped.yaml" - -result: SKIPPED -description: This story should be skipped. - -chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/fixtures/fail_non_existent_variable.yaml b/tools/tests/tester/fixtures/fail_non_existent_variable.yaml index d1f50797a..77e3b8f95 100644 --- a/tools/tests/tester/fixtures/fail_non_existent_variable.yaml +++ b/tools/tests/tester/fixtures/fail_non_existent_variable.yaml @@ -1,6 +1,5 @@ $schema: ../json_schemas/test_story.schema.yaml -skip: false description: | This test story checks that we can create a model group chapters: diff --git a/tools/tests/tester/fixtures/stories/skipped.yaml b/tools/tests/tester/fixtures/stories/skipped.yaml deleted file mode 100644 index 256d65094..000000000 --- a/tools/tests/tester/fixtures/stories/skipped.yaml +++ /dev/null @@ -1,17 +0,0 @@ -$schema: ../../../../../json_schemas/test_story.schema.yaml - -skip: true -description: This story should be skipped. -prologues: - - path: /_cluster/settings - method: GET -epilogues: - - path: /_cluster/settings - method: PUT -chapters: - - synopsis: This chapter should not be executed. - path: /_cat/health - method: GET - - synopsis: This chapter should not be executed. - path: /_cat/indices - method: GET \ No newline at end of file diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index bcb8959f4..90715820e 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -17,12 +17,6 @@ test('passed', async () => { expect(actual).toEqual(expected) }) -test('skipped', async () => { - const actual = await load_actual_evaluation(story_evaluator, 'skipped') - const expected = load_expected_evaluation('skipped') - expect(actual).toEqual(expected) -}) - test('failed/not_found', async () => { const actual = await load_actual_evaluation(story_evaluator, 'failed/not_found') const expected = load_expected_evaluation('failed/not_found') diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index e06802c95..1b12fbe40 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -24,13 +24,12 @@ test('stories folder', async () => { actual_evaluations.push(rest) } - const skipped = load_expected_evaluation('skipped', true) const passed = load_expected_evaluation('passed', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] + const expected_evaluations = [passed, chapter_error, prologue_error, invalid_data, not_found] expect(actual_evaluations).toEqual(expected_evaluations) })