From e1d1844f069c50810345aa89dcc2e03c95e1ffae Mon Sep 17 00:00:00 2001 From: dblock Date: Wed, 14 Aug 2024 11:00:30 -0400 Subject: [PATCH] Added distributions/included and excluded. Signed-off-by: dblock --- TESTING_GUIDE.md | 9 +++-- json_schemas/test_story.schema.yaml | 10 +++++ tests/default/_core/info.yaml | 4 +- tests/default/cat/health.yaml | 8 +++- tests/default/cat/indices.yaml | 8 +++- tests/default/cat/nodeattrs.yaml | 4 +- tests/default/indices/cache.yaml | 4 +- tests/default/indices/dangling.yaml | 4 +- tests/default/indices/forcemerge.yaml | 4 +- tests/default/indices/segments.yaml | 4 +- tests/default/indices/settings.yaml | 4 +- tests/default/ml/model_groups.yaml | 4 +- tests/default/ml/models.yaml | 4 +- tools/src/tester/ChapterEvaluator.ts | 5 ++- tools/src/tester/StoryEvaluator.ts | 40 ++++++++++++++----- .../tester/SupplementalChapterEvaluator.ts | 8 ++-- tools/src/tester/types/story.types.ts | 15 ++++++- .../fixtures/evals/skipped/distributions.yaml | 6 --- .../evals/skipped/distributions/chapters.yaml | 16 ++++++++ .../evals/skipped/distributions/excluded.yaml | 6 +++ .../evals/skipped/distributions/included.yaml | 6 +++ .../skipped/distributions/chapters.yaml | 22 ++++++++++ .../skipped/distributions/excluded.yaml | 10 +++++ .../included.yaml} | 8 ++-- tools/tests/tester/helpers.ts | 13 +++--- tools/tests/tester/integ/TestRunner.test.ts | 25 +++++++----- 26 files changed, 191 insertions(+), 60 deletions(-) delete mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml rename tools/tests/tester/fixtures/stories/skipped/{distributions.yaml => distributions/included.yaml} (51%) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index d35c96aa5..bda054ab7 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -228,13 +228,16 @@ OpenSearch consists of plugins that may or may not be present in various distrib description: Returns basic information about the cluster. ``` -Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support or do not support it. ```yaml description: Test root endpoint. distributions: - - amazon-managed - - opensearch.org + included: + - amazon-managed + - opensearch.org + excluded: + - amazon-serverless chapters: - synopsis: Get server info. path: / diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 928ba4bad..b41a904ef 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -111,6 +111,16 @@ definitions: type: string Distributions: + description: | + The list of distributions that support this API. + type: object + properties: + included: + $ref: '#/definitions/DistributionsList' + excluded: + $ref: '#/definitions/DistributionsList' + + DistributionsList: description: | The list of distributions that support this API. type: array diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 9cc644a6b..f8a360360 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -3,8 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. distributions: - - amazon-managed - - opensearch.org + excluded: + - amazon-serverless chapters: - synopsis: Get server info. path: / diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 196cff4b3..dff956b3e 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -93,7 +93,9 @@ chapters: node.data: '1' - synopsis: Cat in different formats (format=cbor). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/health parameters: @@ -106,7 +108,9 @@ chapters: node.data: '1' - synopsis: Cat in different formats (format=smile). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/health parameters: diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index 6b6130823..2e2ed1711 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -72,7 +72,9 @@ chapters: content_type: application/yaml - synopsis: Cat in different formats (format=cbor). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/indices parameters: @@ -82,7 +84,9 @@ chapters: content_type: application/cbor - synopsis: Cat in different formats (format=smile). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index 01dba1cae..b06b77fb8 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -4,7 +4,9 @@ description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index 2e056a526..a079e25e5 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 0b1b0c784..419338d91 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index cc1608cb6..8fb96e627 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -3,7 +3,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index 0445ebd64..0a481f578 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 80906a7e3..d298e102f 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -26,7 +26,9 @@ chapters: status: 200 - synopsis: Get global settings (cluster_manager_timeout). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index 8c036ae6e..33ad68cbe 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index f92ee82d0..2e0a94524 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index ae73f90a3..dee177ddc 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -87,7 +87,6 @@ export default class ChapterEvaluator { path: `${chapter.method} ${chapter.path}`, overall: { result: overall_result(evaluations) }, request: { parameters: params, request }, - retries, response: { status, payload_body: payload_body_evaluation, @@ -96,6 +95,10 @@ export default class ChapterEvaluator { } } + if (retries !== undefined) { + result.retries = retries + } + if (output_values_evaluation?.output !== undefined) { result.output = output_values_evaluation?.output } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 3f396a3c6..bfed192e7 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -7,7 +7,7 @@ * compatible open source license. */ -import { ChapterRequest, Parameter, type Chapter, type Story, type SupplementalChapter } from './types/story.types' +import { ChapterRequest, Parameter, type Chapter, type Story, type SupplementalChapter } from './types/story.types'; import { type StoryFile, type ChapterEvaluation, Result, type StoryEvaluation, OutputReference } from './types/eval.types' import type ChapterEvaluator from './ChapterEvaluator' import { overall_result } from './helpers' @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } - if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + if (distribution != undefined && story.distributions?.included !== undefined && story.distributions?.included.length > 0 && !story.distributions.included.includes(distribution)) { return { result: Result.SKIPPED, display_path, full_path, description: story.description, - message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + message: `Skipped because distribution ${distribution} is not ${story.distributions.included.length > 1 ? 'one of ' : ''}${story.distributions.included.join(', ')}.` + } + } + + if (distribution != undefined && story.distributions?.excluded !== undefined && story.distributions?.excluded.length > 0 && story.distributions.excluded.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is ${story.distributions.excluded.length > 1 ? 'one of ' : ''}${story.distributions.excluded.join(', ')}.` } } @@ -55,7 +65,8 @@ export default class StoryEvaluator { const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) - return { + + const result: StoryEvaluation = { display_path, full_path, description: story.description, @@ -63,8 +74,14 @@ export default class StoryEvaluator { prologues, epilogues, result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)), - warnings: this.#chapter_warnings(story.chapters) } + + const warnings = this.#chapter_warnings(story.chapters) + if (warnings !== undefined) { + result.warnings = warnings + } + + return result } #chapter_warnings(chapters: Chapter[]): string[] | undefined { @@ -91,13 +108,16 @@ export default class StoryEvaluator { for (const chapter of chapters) { if (dry_run) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) - } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } }) + } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.included.length > 1 ? 'one of ' : ''}${chapter.distributions.included.join(', ')}.` } }) + } else if (distribution != undefined && chapter.distributions?.excluded !== undefined && chapter.distributions?.excluded.length > 0 && chapter.distributions.excluded.includes(distribution)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR @@ -116,7 +136,7 @@ export default class StoryEvaluator { for (const chapter of chapters) { const title = `${chapter.method} ${chapter.path}` if (dry_run) { - evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) } else { const { evaluation, evaluation_error } = await this._supplemental_chapter_evaluator.evaluate(chapter, story_outputs) has_errors = has_errors || evaluation_error diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index e507e9915..faa6c1004 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -12,7 +12,7 @@ import { ChapterOutput } from "./ChapterOutput"; import ChapterReader from "./ChapterReader"; import { StoryOutputs } from "./StoryOutputs"; import { overall_result } from "./helpers"; -import { ChapterEvaluation, Result } from "./types/eval.types"; +import { ChapterEvaluation, Evaluation, Result } from "./types/eval.types"; import { SupplementalChapter } from "./types/story.types"; import { Logger } from "../Logger"; import { to_json } from "../helpers"; @@ -71,9 +71,11 @@ export default class SupplementalChapterEvaluator { const message = message_segments.join('\n') + var overall: Evaluation = { result: Result.ERROR, message } + if (response.error !== undefined) overall.error = response.error as Error + var evaluation: ChapterEvaluation = { - title, - overall: { result: Result.ERROR, message, error: response.error as Error } + title, overall } if (output_values_evaluation.output) { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 83d307b44..e1937b112 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -55,9 +55,9 @@ export type Version = string; * * * This interface was referenced by `Story`'s JSON-Schema - * via the `definition` "Distributions". + * via the `definition` "DistributionsList". */ -export type Distributions = string[]; +export type DistributionsList = string[]; /** * Number of times to retry on error. * @@ -150,6 +150,17 @@ export interface Request { export interface Output { [k: string]: string; } +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export interface Distributions { + included?: DistributionsList; + excluded?: DistributionsList; +} /** * This interface was referenced by `Story`'s JSON-Schema * via the `definition` "ExpectedResponse". diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml deleted file mode 100644 index cc4ef9a19..000000000 --- a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -display_path: skipped/distributions.yaml -full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml - -result: SKIPPED -description: This story should be skipped because of distributions. -message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml new file mode 100644 index 000000000..e392a9d81 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml @@ -0,0 +1,16 @@ +display_path: skipped/distributions/chapters.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml + +description: This story has chapters with distributions. +prologues: [] +epilogues: [] +result: SKIPPED +chapters: + - title: This chapter is excluded because of excluded distributions (opensearch.org). + overall: + message: Skipped because distribution opensearch.org is opensearch.org. + result: SKIPPED + - title: This chapter is excluded because of included distributions (opensearch.org). + overall: + message: Skipped because distribution opensearch.org is not amazon-managed. + result: SKIPPED diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml new file mode 100644 index 000000000..b548402a7 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions/excluded.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml + +result: SKIPPED +description: This story should be skipped because of distributions (excluded). +message: Skipped because distribution opensearch.org is one of opensearch.org, amazon-managed. \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml new file mode 100644 index 000000000..391decc11 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions/included.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml + +result: SKIPPED +description: This story should be skipped because of distributions (included). +message: Skipped because distribution opensearch.org is not amazon-managed. diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml new file mode 100644 index 000000000..527b40dfa --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story has chapters with distributions. +prologues: [] +epilogues: [] +chapters: + - synopsis: This chapter is excluded because of excluded distributions (opensearch.org). + distributions: + excluded: + - opensearch.org + path: /{index} + method: PUT + parameters: + index: index + - synopsis: This chapter is excluded because of included distributions (opensearch.org). + distributions: + included: + - amazon-managed + path: /{index} + method: PUT + parameters: + index: index diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml new file mode 100644 index 000000000..38102bc76 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions (excluded). +distributions: + excluded: + - amazon-managed + - opensearch.org +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml similarity index 51% rename from tools/tests/tester/fixtures/stories/skipped/distributions.yaml rename to tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml index 8efebbb2f..27e8776dc 100644 --- a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml @@ -1,9 +1,9 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml -description: This story should be skipped because of distributions. +description: This story should be skipped because of distributions (included). distributions: - - another - - some + included: + - amazon-managed prologues: [] epilogues: [] chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index d6d7e5639..1a38c4c6b 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -122,12 +122,13 @@ export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation { }) as T } - return { - ...evaluation, - chapters: flatten_chapters(evaluation.chapters), - epilogues: flatten_chapters(evaluation.epilogues), - prologues: flatten_chapters(evaluation.prologues) - } + const result = evaluation + + if (evaluation.chapters !== undefined) result.chapters = flatten_chapters(evaluation.chapters) + if (evaluation.epilogues !== undefined) result.epilogues = flatten_chapters(evaluation.epilogues) + if (evaluation.prologues !== undefined) result.prologues = flatten_chapters(evaluation.prologues) + + return result } export function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Omit & { full_path?: string } { diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 0259c29ac..c1b3e2196 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -28,17 +28,20 @@ test('stories folder', async () => { actual_evaluations.push(rest) } - const passed = load_expected_evaluation('passed', true) - const skipped_semver = load_expected_evaluation('skipped/semver', true) - const skipped_distributions = load_expected_evaluation('skipped/distributions', true) - const not_found = load_expected_evaluation('failed/not_found', true) - const invalid_data = load_expected_evaluation('failed/invalid_data', true) - const chapter_error = load_expected_evaluation('error/chapter_error', true) - const output_error = load_expected_evaluation('error/output_error', true) - const prologue_error = load_expected_evaluation('error/prologue_error', true) - - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] - expect(actual_evaluations).toEqual(expected_evaluations) + const expected_evaluations = [ + 'passed', + 'error/chapter_error', + 'error/output_error', + 'error/prologue_error', + 'failed/invalid_data', + 'failed/not_found', + 'skipped/semver', + 'skipped/distributions/chapters', + 'skipped/distributions/excluded', + 'skipped/distributions/included' + ].map((fixture) => { return load_expected_evaluation(fixture, true) }) + + expect(actual_evaluations).toStrictEqual(expected_evaluations) }) describe('story_files', () => {