diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 333eb7840..5888b88e4 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -11,29 +11,25 @@ import { type Chapter, type ActualResponse } from './types/story.types' import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.types' import { type ParsedOperation } from './types/spec.types' import { overall_result } from './helpers' -import type ChapterReader from './ChapterReader' -import SharedResources from './SharedResources' -import type SpecParser from './SpecParser' -import type SchemaValidator from './SchemaValidator' +import { type OpenAPIV3 } from 'openapi-types' +import ChapterReader from './ChapterReader' +import SpecParser from './SpecParser' +import SchemaValidator from './SchemaValidator' export default class ChapterEvaluator { + spec: OpenAPIV3.Document chapter: Chapter skip_payload_evaluation: boolean = false - spec_parser: SpecParser - chapter_reader: ChapterReader - schema_validator: SchemaValidator - constructor (chapter: Chapter) { + constructor (chapter: Chapter, spec: OpenAPIV3.Document) { this.chapter = chapter - this.spec_parser = SharedResources.get_instance().spec_parser - this.chapter_reader = SharedResources.get_instance().chapter_reader - this.schema_validator = SharedResources.get_instance().schema_validator + this.spec = spec } async evaluate (skip: boolean): Promise { if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } } - const response = await this.chapter_reader.read(this.chapter) - const operation = this.spec_parser.locate_operation(this.chapter) + const response = await new ChapterReader().read(this.chapter) + const operation = new SpecParser(this.spec).locate_operation(this.chapter) if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } } const params = this.#evaluate_parameters(operation) const request_body = this.#evaluate_request_body(operation) @@ -51,7 +47,7 @@ export default class ChapterEvaluator { return Object.fromEntries(Object.entries(this.chapter.parameters ?? {}).map(([name, parameter]) => { const schema = operation.parameters[name]?.schema if (schema == null) return [name, { result: Result.FAILED, message: `Schema for "${name}" parameter not found.` }] - const evaluation = this.schema_validator.validate(schema, parameter) + const evaluation = new SchemaValidator(this.spec).validate(schema, parameter) return [name, evaluation] })) } @@ -61,7 +57,7 @@ export default class ChapterEvaluator { const content_type = this.chapter.request_body.content_type ?? 'application/json' const schema = operation.requestBody?.content[content_type]?.schema if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` } - return this.schema_validator.validate(schema, this.chapter.request_body?.payload ?? {}) + return new SchemaValidator(this.spec).validate(schema, this.chapter.request_body?.payload ?? {}) } #evaluate_status (response: ActualResponse): Evaluation { @@ -82,6 +78,6 @@ export default class ChapterEvaluator { const schema = content?.schema if (schema == null && content != null) return { result: Result.PASSED } if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` } - return this.schema_validator.validate(schema, response.payload) + return new SchemaValidator(this.spec).validate(schema, response.payload) } } diff --git a/tools/src/tester/SharedResources.ts b/tools/src/tester/SharedResources.ts deleted file mode 100644 index cf56dc87e..000000000 --- a/tools/src/tester/SharedResources.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import type ChapterReader from './ChapterReader' -import type SchemaValidator from './SchemaValidator' -import type SpecParser from './SpecParser' - -interface Resources { - chapter_reader: ChapterReader - schema_validator: SchemaValidator - spec_parser: SpecParser -} - -export default class SharedResources { - private static instance: SharedResources | undefined - chapter_reader: ChapterReader - schema_validator: SchemaValidator - spec_parser: SpecParser - - private constructor (resources: Resources) { - this.chapter_reader = resources.chapter_reader - this.schema_validator = resources.schema_validator - this.spec_parser = resources.spec_parser - } - - public static create_instance (resources: Resources): void { - if (SharedResources.instance) throw new Error('SharedResources instance has already been created.') - SharedResources.instance = new SharedResources(resources) - } - - public static get_instance (): SharedResources { - if (SharedResources.instance) return SharedResources.instance - throw new Error('SharedResources instance has not been created.') - } -} diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index bafe2db43..221e5f00b 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -7,12 +7,12 @@ * compatible open source license. */ +import { type OpenAPIV3 } from 'openapi-types' import { type Chapter, type Story, type SupplementalChapter } from './types/story.types' import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/eval.types' import ChapterEvaluator from './ChapterEvaluator' -import type ChapterReader from './ChapterReader' -import SharedResources from './SharedResources' import { overall_result } from './helpers' +import ChapterReader from './ChapterReader' export interface StoryFile { display_path: string @@ -21,19 +21,19 @@ export interface StoryFile { } export default class StoryEvaluator { + spec: OpenAPIV3.Document dry_run: boolean story: Story display_path: string full_path: string has_errors: boolean = false - chapter_reader: ChapterReader - constructor (story_file: StoryFile, dry_run?: boolean) { + constructor (story_file: StoryFile, spec: OpenAPIV3.Document, dry_run?: boolean) { + this.spec = spec this.dry_run = dry_run ?? false this.story = story_file.story this.display_path = story_file.display_path this.full_path = story_file.full_path - this.chapter_reader = SharedResources.get_instance().chapter_reader } async evaluate (): Promise { @@ -67,7 +67,7 @@ export default class StoryEvaluator { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { - const evaluator = new ChapterEvaluator(chapter) + const evaluator = new ChapterEvaluator(chapter, this.spec) const evaluation = await evaluator.evaluate(this.has_errors) this.has_errors = this.has_errors || evaluation.overall.result === Result.ERROR evaluations.push(evaluation) @@ -83,7 +83,7 @@ export default class StoryEvaluator { if (this.dry_run) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { - const response = await this.chapter_reader.read(chapter) + const response = await new ChapterReader().read(chapter) const status = chapter.status ?? [200, 201] if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } }) else { diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts index 94b00f543..5f26d1776 100644 --- a/tools/src/tester/TestsRunner.ts +++ b/tools/src/tester/TestsRunner.ts @@ -8,16 +8,12 @@ */ import { type OpenAPIV3 } from 'openapi-types' -import SpecParser from './SpecParser' -import ChapterReader from './ChapterReader' -import SchemaValidator from './SchemaValidator' import StoryEvaluator, { type StoryFile } from './StoryEvaluator' import fs from 'fs' import { type Story } from './types/story.types' import { read_yaml } from '../../helpers' import { Result, type StoryEvaluation } from './types/eval.types' import ResultsDisplayer, { type TestRunOptions, type DisplayOptions } from './ResultsDisplayer' -import SharedResources from './SharedResources' import { resolve, basename } from 'path' type TestsRunnerOptions = TestRunOptions & DisplayOptions & Record @@ -25,15 +21,12 @@ type TestsRunnerOptions = TestRunOptions & DisplayOptions & Record export default class TestsRunner { path: string // Path to a story file or a directory containing story files opts: TestsRunnerOptions + spec: OpenAPIV3.Document constructor (spec: OpenAPIV3.Document, path: string, opts: TestsRunnerOptions) { this.path = resolve(path) this.opts = opts - - const chapter_reader = new ChapterReader() - const spec_parser = new SpecParser(spec) - const schema_validator = new SchemaValidator(spec) - SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser }) + this.spec = spec } async run (debug: boolean = false): Promise { @@ -41,7 +34,7 @@ export default class TestsRunner { const story_files = this.#collect_story_files(this.path, '', '') const evaluations: StoryEvaluation[] = [] for (const story_file of this.#sort_story_files(story_files)) { - const evaluator = new StoryEvaluator(story_file, this.opts.dry_run) + const evaluator = new StoryEvaluator(story_file, this.spec, this.opts.dry_run) const evaluation = await evaluator.evaluate() const displayer = new ResultsDisplayer(evaluation, this.opts) if (debug) evaluations.push(evaluation) diff --git a/tools/src/tester/start.ts b/tools/src/tester/start.ts index 797602158..995db223f 100644 --- a/tools/src/tester/start.ts +++ b/tools/src/tester/start.ts @@ -32,6 +32,7 @@ const options = { // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge() const runner = new TestsRunner(spec, opts.testsPath, options) void runner.run().then(() => { _.noop() }) diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts index 3cc57aa90..0fa12bdd6 100644 --- a/tools/tests/tester/StoryEvaluator.test.ts +++ b/tools/tests/tester/StoryEvaluator.test.ts @@ -7,45 +7,52 @@ * compatible open source license. */ -import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' +import { load_actual_evaluation, load_expected_evaluation } from './helpers' import { read_yaml } from '../../helpers' import { type OpenAPIV3 } from 'openapi-types' -const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') -create_shared_resources(spec as OpenAPIV3.Document) - -test('passed', async () => { - const actual = await load_actual_evaluation('passed') - const expected = await load_expected_evaluation('passed') - expect(actual).toEqual(expected) -}) - -test('skipped', async () => { - const actual = await load_actual_evaluation('skipped') - const expected = await load_expected_evaluation('skipped') - expect(actual).toEqual(expected) -}) - -test('failed/not_found', async () => { - const actual = await load_actual_evaluation('failed/not_found') - const expected = await load_expected_evaluation('failed/not_found') - expect(actual).toEqual(expected) -}) - -test('failed/invalid_data', async () => { - const actual = await load_actual_evaluation('failed/invalid_data') - const expected = await load_expected_evaluation('failed/invalid_data') - expect(actual).toEqual(expected) -}) - -test('error/prologue_error', async () => { - const actual = await load_actual_evaluation('error/prologue_error') - const expected = await load_expected_evaluation('error/prologue_error') - expect(actual).toEqual(expected) -}) - -test('error/chapter_error', async () => { - const actual = await load_actual_evaluation('error/chapter_error') - const expected = await load_expected_evaluation('error/chapter_error') - expect(actual).toEqual(expected) +describe('StoryEvaluator', () => { + let spec: OpenAPIV3.Document + + beforeAll(() => { + // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml + process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + }) + + test('passed', async () => { + const actual = await load_actual_evaluation('passed', spec) + const expected = await load_expected_evaluation('passed') + expect(actual).toEqual(expected) + }) + + test('skipped', async () => { + const actual = await load_actual_evaluation('skipped', spec) + const expected = await load_expected_evaluation('skipped') + expect(actual).toEqual(expected) + }) + + test('failed/not_found', async () => { + const actual = await load_actual_evaluation('failed/not_found', spec) + const expected = await load_expected_evaluation('failed/not_found') + expect(actual).toEqual(expected) + }) + + test('failed/invalid_data', async () => { + const actual = await load_actual_evaluation('failed/invalid_data', spec) + const expected = await load_expected_evaluation('failed/invalid_data') + expect(actual).toEqual(expected) + }) + + test('error/prologue_error', async () => { + const actual = await load_actual_evaluation('error/prologue_error', spec) + const expected = await load_expected_evaluation('error/prologue_error') + expect(actual).toEqual(expected) + }) + + test('error/chapter_error', async () => { + const actual = await load_actual_evaluation('error/chapter_error', spec) + const expected = await load_expected_evaluation('error/chapter_error') + expect(actual).toEqual(expected) + }) }) diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/tests/tester/TestsRunner.test.ts index d2290d808..d3e365674 100644 --- a/tools/tests/tester/TestsRunner.test.ts +++ b/tools/tests/tester/TestsRunner.test.ts @@ -12,25 +12,33 @@ import TestsRunner from '../../src/tester/TestsRunner' import { type OpenAPIV3 } from 'openapi-types' import { load_expected_evaluation, scrub_errors } from './helpers' -test('stories folder', async () => { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') - const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {}) - const actual_evaluations = await runner.run(true) as any[] - for (const evaluation of actual_evaluations) scrub_errors(evaluation) - for (const evaluation of actual_evaluations) { - expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() - delete evaluation.full_path - } +describe('stories folder', () => { + let spec: OpenAPIV3.Document - const skipped = await load_expected_evaluation('skipped', true) - const passed = await load_expected_evaluation('passed', true) - const not_found = await load_expected_evaluation('failed/not_found', true) - const invalid_data = await load_expected_evaluation('failed/invalid_data', true) - const chapter_error = await load_expected_evaluation('error/chapter_error', true) - const prologue_error = await load_expected_evaluation('error/prologue_error', true) + beforeAll(() => { + // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml + process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + }) - const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] - expect(actual_evaluations).toEqual(expected_evaluations) + test('results', async () => { + const runner = new TestsRunner(spec, 'tools/tests/tester/fixtures/stories', {}) + const actual_evaluations = await runner.run(true) as any[] + + for (const evaluation of actual_evaluations) { + scrub_errors(evaluation) + expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() + delete evaluation.full_path + } + + const skipped = await load_expected_evaluation('skipped', true) + const passed = await load_expected_evaluation('passed', true) + const not_found = await load_expected_evaluation('failed/not_found', true) + const invalid_data = await load_expected_evaluation('failed/invalid_data', true) + const chapter_error = await load_expected_evaluation('error/chapter_error', true) + const prologue_error = await load_expected_evaluation('error/prologue_error', true) + + const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] + expect(actual_evaluations).toEqual(expected_evaluations) + }) }) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 99e274e48..d273d6b23 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -7,26 +7,13 @@ * compatible open source license. */ -import ChapterReader from '../../src/tester/ChapterReader' -import SpecParser from '../../src/tester/SpecParser' -import SchemaValidator from '../../src/tester/SchemaValidator' -import SharedResources from '../../src/tester/SharedResources' -import { type OpenAPIV3 } from 'openapi-types' import YAML from 'yaml' +import { type OpenAPIV3 } from 'openapi-types' import type { StoryEvaluation } from '../../src/tester/types/eval.types' import type { Story } from '../../src/tester/types/story.types' import { read_yaml } from '../../helpers' import StoryEvaluator from '../../src/tester/StoryEvaluator' -export function create_shared_resources (spec: any): void { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const chapter_reader = new ChapterReader() - const spec_parser = new SpecParser(spec as OpenAPIV3.Document) - const schema_validator = new SchemaValidator(spec as OpenAPIV3.Document) - SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser }) -} - export function print_yaml (obj: any): void { console.log(YAML.stringify(obj, { indent: 2, singleQuote: true, lineWidth: undefined })) } @@ -45,11 +32,11 @@ export async function load_expected_evaluation (name: string, exclude_full_path: return expected } -export async function load_actual_evaluation (name: string): Promise { +export async function load_actual_evaluation (name: string, spec: OpenAPIV3.Document): Promise { const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`) const display_path = `${name}.yaml` const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml` - const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate() + const actual = await new StoryEvaluator({ display_path, full_path, story }, spec).evaluate() scrub_errors(actual) return actual }