diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 652c300fb..76cb8672d 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -1,27 +1,40 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + import { Option } from '@commander-js/extra-typings' import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios' import * as https from 'node:https' import { sleep } from '../helpers' +const DEFAULT_URL = 'https://localhost:9200' +const DEFAULT_USER = 'admin' +const DEFAULT_INSECURE = false + export const OPENSEARCH_URL_OPTION = new Option('--opensearch-url ', 'URL at which the OpenSearch cluster is accessible') - .default('https://localhost:9200') + .default(DEFAULT_URL) .env('OPENSEARCH_URL') export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username ', 'username to use when authenticating with OpenSearch') - .default('admin') + .default(DEFAULT_USER) .env('OPENSEARCH_USERNAME') export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password ', 'password to use when authenticating with OpenSearch') .env('OPENSEARCH_PASSWORD') export const OPENSEARCH_INSECURE_OPTION = new Option('--opensearch-insecure', 'disable SSL/TLS certificate verification when connecting to OpenSearch') - .default(false) + .default(DEFAULT_INSECURE) export interface OpenSearchHttpClientOptions { - url: string + url?: string username?: string password?: string - insecure: boolean + insecure?: boolean } export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize}`]: OpenSearchHttpClientOptions[K] } @@ -56,16 +69,16 @@ export interface OpenSearchInfo { export class OpenSearchHttpClient { private readonly _axios: AxiosInstance - constructor (opts: OpenSearchHttpClientOptions) { + constructor (opts?: OpenSearchHttpClientOptions) { this._axios = axios.create({ - baseURL: opts.url, - auth: opts.username !== undefined && opts.password !== undefined + baseURL: opts?.url ?? DEFAULT_URL, + auth: opts?.username !== undefined && opts.password !== undefined ? { username: opts.username, password: opts.password } : undefined, - httpsAgent: new https.Agent({ rejectUnauthorized: !opts.insecure }) + httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }) }) } diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts new file mode 100644 index 000000000..99d3f8fc7 --- /dev/null +++ b/tools/src/tester/ResultLogger.ts @@ -0,0 +1,104 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' +import { overall_result } from './helpers' +import * as ansi from './Ansi' + +export interface ResultLogger { + log: (evaluation: StoryEvaluation) => void +} + +export class NoOpResultLogger implements ResultLogger { + log (_: StoryEvaluation): void { } +} + +export class ConsoleResultLogger implements ResultLogger { + private readonly _tab_width: number + private readonly _verbose: boolean + + constructor (tab_width: number = 4, verbose: boolean = false) { + this._tab_width = tab_width + this._verbose = verbose + } + + log (evaluation: StoryEvaluation): void { + this.#log_story(evaluation) + this.#log_chapters(evaluation.prologues ?? [], 'PROLOGUES') + this.#log_chapters(evaluation.chapters ?? [], 'CHAPTERS') + this.#log_chapters(evaluation.epilogues ?? [], 'EPILOGUES') + console.log('\n') + } + + #log_story ({ result, full_path, description, display_path }: StoryEvaluation): void { + this.#log_evaluation({ result, message: full_path }, ansi.cyan(ansi.b(description ?? display_path))) + } + + #log_chapters (evaluations: ChapterEvaluation[], title: string): void { + if (evaluations.length === 0) return + const result = overall_result(evaluations.map(e => e.overall)) + if (!this._verbose && (result === Result.SKIPPED || result === Result.PASSED)) return + this.#log_evaluation({ result }, title, this._tab_width) + for (const evaluation of evaluations) this.#log_chapter(evaluation) + } + + #log_chapter (chapter: ChapterEvaluation): void { + this.#log_evaluation(chapter.overall, ansi.i(chapter.title), this._tab_width * 2) + this.#log_parameters(chapter.request?.parameters ?? {}) + this.#log_request_body(chapter.request?.request_body) + this.#log_status(chapter.response?.status) + this.#log_payload(chapter.response?.payload) + } + + #log_parameters (parameters: Record): void { + if (Object.keys(parameters).length === 0) return + const result = overall_result(Object.values(parameters)) + this.#log_evaluation({ result }, 'PARAMETERS', this._tab_width * 3) + for (const [name, evaluation] of Object.entries(parameters)) { + this.#log_evaluation(evaluation, name, this._tab_width * 4) + } + } + + #log_request_body (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'REQUEST BODY', this._tab_width * 3) + } + + #log_status (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'RESPONSE STATUS', this._tab_width * 3) + } + + #log_payload (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD', this._tab_width * 3) + } + + #log_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void { + const result = ansi.padding(this.#result(evaluation.result), 0, prefix) + const message = evaluation.message != null ? `${ansi.gray('(' + evaluation.message + ')')}` : '' + console.log(`${result} ${title} ${message}`) + if (evaluation.error && this._verbose) { + console.log('-'.repeat(100)) + console.error(evaluation.error) + console.log('-'.repeat(100)) + } + } + + #result (r: Result): string { + const text = ansi.padding(r, 7) + switch (r) { + case Result.PASSED: return ansi.green(text) + case Result.SKIPPED: return ansi.yellow(text) + case Result.FAILED: return ansi.magenta(text) + case Result.ERROR: return ansi.red(text) + default: return ansi.gray(text) + } + } +} diff --git a/tools/src/tester/ResultsDisplay.ts b/tools/src/tester/ResultsDisplay.ts deleted file mode 100644 index e7209431e..000000000 --- a/tools/src/tester/ResultsDisplay.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' -import { overall_result } from './helpers' -import * as ansi from './Ansi' - -export interface DisplayOptions { - tab_width?: number - verbose?: boolean -} - -export default class ResultsDisplay { - private readonly _tab_width: number - private readonly _verbose: boolean - - constructor (opts: DisplayOptions) { - this._tab_width = opts.tab_width ?? 4 - this._verbose = opts.verbose ?? false - } - - display (evaluation: StoryEvaluation): void { - this.#display_story(evaluation) - this.#display_chapters(evaluation.prologues ?? [], 'PROLOGUES') - this.#display_chapters(evaluation.chapters ?? [], 'CHAPTERS') - this.#display_chapters(evaluation.epilogues ?? [], 'EPILOGUES') - console.log('\n') - } - - #display_story ({ result, full_path, description, display_path }: StoryEvaluation): void { - this.#display_evaluation({ result, message: full_path }, ansi.cyan(ansi.b(description ?? display_path))) - } - - #display_chapters (evaluations: ChapterEvaluation[], title: string): void { - if (evaluations.length === 0) return - const result = overall_result(evaluations.map(e => e.overall)) - if (!this._verbose && (result === Result.SKIPPED || result === Result.PASSED)) return - this.#display_evaluation({ result }, title, this._tab_width) - for (const evaluation of evaluations) this.#display_chapter(evaluation) - } - - #display_chapter (chapter: ChapterEvaluation): void { - this.#display_evaluation(chapter.overall, ansi.i(chapter.title), this._tab_width * 2) - this.#display_parameters(chapter.request?.parameters ?? {}) - this.#display_request_body(chapter.request?.request_body) - this.#display_status(chapter.response?.status) - this.#display_payload(chapter.response?.payload) - } - - #display_parameters (parameters: Record): void { - if (Object.keys(parameters).length === 0) return - const result = overall_result(Object.values(parameters)) - this.#display_evaluation({ result }, 'PARAMETERS', this._tab_width * 3) - for (const [name, evaluation] of Object.entries(parameters)) { - this.#display_evaluation(evaluation, name, this._tab_width * 4) - } - } - - #display_request_body (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'REQUEST BODY', this._tab_width * 3) - } - - #display_status (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'RESPONSE STATUS', this._tab_width * 3) - } - - #display_payload (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'RESPONSE PAYLOAD', this._tab_width * 3) - } - - #display_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void { - const result = ansi.padding(this.#result(evaluation.result), 0, prefix) - const message = evaluation.message != null ? `${ansi.gray('(' + evaluation.message + ')')}` : '' - console.log(`${result} ${title} ${message}`) - if (evaluation.error && this._verbose) { - console.log('-'.repeat(100)) - console.error(evaluation.error) - console.log('-'.repeat(100)) - } - } - - #result (r: Result): string { - const text = ansi.padding(r, 7) - switch (r) { - case Result.PASSED: return ansi.green(text) - case Result.SKIPPED: return ansi.yellow(text) - case Result.FAILED: return ansi.magenta(text) - case Result.ERROR: return ansi.red(text) - default: return ansi.gray(text) - } - } -} diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index aef94eb8f..9769195ac 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -22,15 +22,13 @@ export interface StoryFile { export default class StoryEvaluator { private readonly _chapter_reader: ChapterReader private readonly _chapter_evaluator: ChapterEvaluator - private readonly _dry_run: boolean - constructor (chapter_reader: ChapterReader, chapter_evaluator: ChapterEvaluator, dry_run: boolean) { + constructor (chapter_reader: ChapterReader, chapter_evaluator: ChapterEvaluator) { this._chapter_reader = chapter_reader this._chapter_evaluator = chapter_evaluator - this._dry_run = dry_run } - async evaluate ({ story, display_path, full_path }: StoryFile): Promise { + async evaluate ({ story, display_path, full_path }: StoryFile, dry_run: boolean = false): Promise { if (story.skip) { return { result: Result.SKIPPED, @@ -40,9 +38,9 @@ export default class StoryEvaluator { chapters: [] } } - const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? []) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors) - const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? []) + const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run) + const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run) return { display_path, full_path, @@ -54,10 +52,10 @@ export default class StoryEvaluator { } } - async #evaluate_chapters (chapters: Chapter[], has_errors: boolean): Promise { + async #evaluate_chapters (chapters: Chapter[], has_errors: boolean, dry_run: boolean): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { - if (this._dry_run) { + if (dry_run) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { @@ -69,12 +67,12 @@ export default class StoryEvaluator { return evaluations } - async #evaluate_supplemental_chapters (chapters: SupplementalChapter[]): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> { + async #evaluate_supplemental_chapters (chapters: SupplementalChapter[], dry_run: boolean): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> { let has_errors = false const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { const title = `${chapter.method} ${chapter.path}` - if (this._dry_run) { + if (dry_run) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { const response = await this._chapter_reader.read(chapter) diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts new file mode 100644 index 000000000..fa88306ab --- /dev/null +++ b/tools/src/tester/TestRunner.ts @@ -0,0 +1,66 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import type StoryEvaluator from './StoryEvaluator' +import { type StoryFile } from './StoryEvaluator' +import fs from 'fs' +import { type Story } from './types/story.types' +import { read_yaml } from '../../helpers' +import { Result, type StoryEvaluation } from './types/eval.types' +import { type ResultLogger } from './ResultLogger' +import { basename, resolve } from 'path' + +export default class TestRunner { + private readonly _story_evaluator: StoryEvaluator + private readonly _result_logger: ResultLogger + + constructor (story_evaluator: StoryEvaluator, result_logger: ResultLogger) { + this._story_evaluator = story_evaluator + this._result_logger = result_logger + } + + async run (story_path: string, dry_run: boolean = false): Promise<{ evaluations: StoryEvaluation[], failed: boolean }> { + let failed = false + const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', '')) + const evaluations: StoryEvaluation[] = [] + for (const story_file of story_files) { + const evaluation = await this._story_evaluator.evaluate(story_file, dry_run) + evaluations.push(evaluation) + this._result_logger.log(evaluation) + if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true + } + return { evaluations, failed } + } + + #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { + const path = file === '' ? folder : `${folder}/${file}` + const next_prefix = prefix === '' ? file : `${prefix}/${file}` + if (fs.statSync(path).isFile()) { + const story: Story = read_yaml(path) + return [{ + display_path: next_prefix === '' ? basename(path) : next_prefix, + full_path: path, + story + }] + } else { + return fs.readdirSync(path).flatMap(next_file => { + return this.#collect_story_files(path, next_file, next_prefix) + }) + } + } + + #sort_story_files (story_files: StoryFile[]): StoryFile[] { + return story_files.sort(({ display_path: a }, { display_path: b }) => { + const a_depth = a.split('/').length + const b_depth = b.split('/').length + if (a_depth !== b_depth) return a_depth - b_depth + return a.localeCompare(b) + }) + } +} diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts deleted file mode 100644 index 651a607f3..000000000 --- a/tools/src/tester/TestsRunner.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { type OpenAPIV3 } from 'openapi-types' -import SpecParser from './SpecParser' -import ChapterReader from './ChapterReader' -import SchemaValidator from './SchemaValidator' -import StoryEvaluator, { type StoryFile } from './StoryEvaluator' -import fs from 'fs' -import { type Story } from './types/story.types' -import { read_yaml } from '../../helpers' -import { Result, type StoryEvaluation } from './types/eval.types' -import ResultsDisplay, { type DisplayOptions } from './ResultsDisplay' -import { basename, resolve } from 'path' -import ChapterEvaluator from './ChapterEvaluator' -import { OpenSearchHttpClient, type OpenSearchHttpClientOptions } from '../OpenSearchHttpClient' - -interface TestsRunnerOptions { - dry_run?: boolean - display: DisplayOptions - opensearch: OpenSearchHttpClientOptions -} - -export default class TestsRunner { - private readonly _http_client: OpenSearchHttpClient - private readonly _story_evaluator: StoryEvaluator - private readonly _results_display: ResultsDisplay - - constructor (spec: OpenAPIV3.Document, opts: TestsRunnerOptions) { - this._http_client = new OpenSearchHttpClient(opts.opensearch) - const chapter_reader = new ChapterReader(this._http_client) - const chapter_evaluator = new ChapterEvaluator(new SpecParser(spec), chapter_reader, new SchemaValidator(spec)) - this._story_evaluator = new StoryEvaluator(chapter_reader, chapter_evaluator, opts.dry_run ?? false) - this._results_display = new ResultsDisplay(opts.display) - } - - async run (story_path: string, debug: boolean = false): Promise { - let failed = false - const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', '')) - const evaluations: StoryEvaluation[] = [] - for (const story_file of story_files) { - const evaluation = await this._story_evaluator.evaluate(story_file) - evaluations.push(evaluation) - if (!debug) { - this._results_display.display(evaluation) - } - if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true - } - if (failed && !debug) process.exit(1) - return evaluations - } - - #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { - const path = file === '' ? folder : `${folder}/${file}` - const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (fs.statSync(path).isFile()) { - const story: Story = read_yaml(path) - return [{ - display_path: next_prefix === '' ? basename(path) : next_prefix, - full_path: path, - story - }] - } else { - return fs.readdirSync(path).flatMap(next_file => { - return this.#collect_story_files(path, next_file, next_prefix) - }) - } - } - - #sort_story_files (story_files: StoryFile[]): StoryFile[] { - return story_files.sort((a, b) => { - const a_depth = a.display_path.split('/').length - const b_depth = b.display_path.split('/').length - if (a_depth !== b_depth) return a_depth - b_depth - return a.display_path.localeCompare(b.display_path) - }) - } -} diff --git a/tools/src/tester/start.ts b/tools/src/tester/start.ts index 827231279..11c2994cf 100644 --- a/tools/src/tester/start.ts +++ b/tools/src/tester/start.ts @@ -9,24 +9,34 @@ import OpenApiMerger from '../merger/OpenApiMerger' import { LogLevel } from '../Logger' -import TestsRunner from './TestsRunner' +import TestRunner from './TestRunner' import { Command, Option } from '@commander-js/extra-typings' -import _ from 'lodash' import { get_opensearch_opts_from_cli, OPENSEARCH_INSECURE_OPTION, OPENSEARCH_PASSWORD_OPTION, OPENSEARCH_URL_OPTION, - OPENSEARCH_USERNAME_OPTION + OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient } from '../OpenSearchHttpClient' +import ChapterReader from './ChapterReader' +import ChapterEvaluator from './ChapterEvaluator' +import SpecParser from './SpecParser' +import SchemaValidator from './SchemaValidator' +import StoryEvaluator from './StoryEvaluator' +import { ConsoleResultLogger } from './ResultLogger' +import * as process from 'node:process' const command = new Command() .description('Run test stories against the OpenSearch spec.') .addOption(new Option('--spec, --spec-path ', 'path to the root folder of the multi-file spec').default('./spec')) .addOption(new Option('--tests, --tests-path ', 'path to the root folder of the tests').default('./tests')) - .addOption(new Option('--tab-width ', 'tab width for displayed results').default('4')) - .addOption(new Option('--verbose', 'whether to print the full stack trace of errors')) - .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests')) + .addOption( + new Option('--tab-width ', 'tab width for displayed results') + .default(4) + .argParser((v, _) => Number.parseInt(v)) + ) + .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) + .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(OPENSEARCH_URL_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) @@ -36,17 +46,18 @@ const command = new Command() const opts = command.opts() -const tests_runner_options = { - dry_run: opts.dryRun, - display: { - verbose: opts.verbose ?? false, - tab_width: Number.parseInt(opts.tabWidth) - }, - opensearch: get_opensearch_opts_from_cli(opts) -} - -// The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml -process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge() -const runner = new TestsRunner(spec, tests_runner_options) -void runner.run(opts.testsPath).then(() => { _.noop() }) + +const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts)) +const chapter_reader = new ChapterReader(http_client) +const chapter_evaluator = new ChapterEvaluator(new SpecParser(spec), chapter_reader, new SchemaValidator(spec)) +const story_evaluator = new StoryEvaluator(chapter_reader, chapter_evaluator) +const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) +const runner = new TestRunner(story_evaluator, result_logger) + +runner.run(opts.testsPath, opts.dryRun) + .then( + ({ failed }) => { + if (failed) process.exit(1) + }, + err => { throw err }) diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts index 3cc57aa90..8d220912e 100644 --- a/tools/tests/tester/StoryEvaluator.test.ts +++ b/tools/tests/tester/StoryEvaluator.test.ts @@ -7,45 +7,42 @@ * compatible open source license. */ -import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' -import { read_yaml } from '../../helpers' -import { type OpenAPIV3 } from 'openapi-types' +import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from './helpers' -const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') -create_shared_resources(spec as OpenAPIV3.Document) +const { story_evaluator } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') test('passed', async () => { - const actual = await load_actual_evaluation('passed') + const actual = await load_actual_evaluation(story_evaluator, 'passed') const expected = await load_expected_evaluation('passed') expect(actual).toEqual(expected) }) test('skipped', async () => { - const actual = await load_actual_evaluation('skipped') + const actual = await load_actual_evaluation(story_evaluator, 'skipped') const expected = await load_expected_evaluation('skipped') expect(actual).toEqual(expected) }) test('failed/not_found', async () => { - const actual = await load_actual_evaluation('failed/not_found') + const actual = await load_actual_evaluation(story_evaluator, 'failed/not_found') const expected = await load_expected_evaluation('failed/not_found') expect(actual).toEqual(expected) }) test('failed/invalid_data', async () => { - const actual = await load_actual_evaluation('failed/invalid_data') + const actual = await load_actual_evaluation(story_evaluator, 'failed/invalid_data') const expected = await load_expected_evaluation('failed/invalid_data') expect(actual).toEqual(expected) }) test('error/prologue_error', async () => { - const actual = await load_actual_evaluation('error/prologue_error') + const actual = await load_actual_evaluation(story_evaluator, 'error/prologue_error') const expected = await load_expected_evaluation('error/prologue_error') expect(actual).toEqual(expected) }) test('error/chapter_error', async () => { - const actual = await load_actual_evaluation('error/chapter_error') + const actual = await load_actual_evaluation(story_evaluator, 'error/chapter_error') const expected = await load_expected_evaluation('error/chapter_error') expect(actual).toEqual(expected) }) diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/tests/tester/TestRunner.test.ts similarity index 58% rename from tools/tests/tester/TestsRunner.test.ts rename to tools/tests/tester/TestRunner.test.ts index d2290d808..dced6a9bf 100644 --- a/tools/tests/tester/TestsRunner.test.ts +++ b/tools/tests/tester/TestRunner.test.ts @@ -7,20 +7,19 @@ * compatible open source license. */ -import { read_yaml } from '../../helpers' -import TestsRunner from '../../src/tester/TestsRunner' -import { type OpenAPIV3 } from 'openapi-types' -import { load_expected_evaluation, scrub_errors } from './helpers' +import { construct_tester_components, load_expected_evaluation, scrub_errors } from './helpers' test('stories folder', async () => { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') - const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {}) - const actual_evaluations = await runner.run(true) as any[] - for (const evaluation of actual_evaluations) scrub_errors(evaluation) + const { test_runner } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + const result = await test_runner.run('tools/tests/tester/fixtures/stories', true) + + expect(result.failed).toBeTruthy() + + const actual_evaluations: any[] = result.evaluations + for (const evaluation of actual_evaluations) { expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() + scrub_errors(evaluation) delete evaluation.full_path } diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 99e274e48..0c1ffdfe6 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -7,24 +7,51 @@ * compatible open source license. */ -import ChapterReader from '../../src/tester/ChapterReader' -import SpecParser from '../../src/tester/SpecParser' -import SchemaValidator from '../../src/tester/SchemaValidator' -import SharedResources from '../../src/tester/SharedResources' -import { type OpenAPIV3 } from 'openapi-types' import YAML from 'yaml' import type { StoryEvaluation } from '../../src/tester/types/eval.types' import type { Story } from '../../src/tester/types/story.types' import { read_yaml } from '../../helpers' import StoryEvaluator from '../../src/tester/StoryEvaluator' +import SpecParser from '../../src/tester/SpecParser' +import ChapterReader from '../../src/tester/ChapterReader' +import SchemaValidator from '../../src/tester/SchemaValidator' +import ChapterEvaluator from '../../src/tester/ChapterEvaluator' +import { OpenSearchHttpClient } from '../../src/OpenSearchHttpClient' +import { type OpenAPIV3 } from 'openapi-types' +import TestRunner from '../../src/tester/TestRunner' +import { NoOpResultLogger, type ResultLogger } from '../../src/tester/ResultLogger' -export function create_shared_resources (spec: any): void { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const chapter_reader = new ChapterReader() - const spec_parser = new SpecParser(spec as OpenAPIV3.Document) - const schema_validator = new SchemaValidator(spec as OpenAPIV3.Document) - SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser }) +export function construct_tester_components (spec_path: string): { + specification: OpenAPIV3.Document + spec_parser: SpecParser + opensearch_http_client: OpenSearchHttpClient + chapter_reader: ChapterReader + schema_validator: SchemaValidator + chapter_evaluator: ChapterEvaluator + story_evaluator: StoryEvaluator + result_logger: ResultLogger + test_runner: TestRunner +} { + const specification: OpenAPIV3.Document = read_yaml(spec_path) + const spec_parser = new SpecParser(specification) + const opensearch_http_client = new OpenSearchHttpClient() + const chapter_reader = new ChapterReader(opensearch_http_client) + const schema_validator = new SchemaValidator(specification) + const chapter_evaluator = new ChapterEvaluator(spec_parser, chapter_reader, schema_validator) + const story_evaluator = new StoryEvaluator(chapter_reader, chapter_evaluator) + const result_logger = new NoOpResultLogger() + const test_runner = new TestRunner(story_evaluator, result_logger) + return { + specification, + spec_parser, + opensearch_http_client, + chapter_reader, + schema_validator, + chapter_evaluator, + story_evaluator, + result_logger, + test_runner + } } export function print_yaml (obj: any): void { @@ -33,9 +60,9 @@ export function print_yaml (obj: any): void { export function scrub_errors (obj: any): void { for (const key in obj) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - if (key === 'error') obj[key] = obj[key].message - else if (typeof obj[key] === 'object') scrub_errors(obj[key]) + if (typeof obj[key] !== 'object') continue + if (key === 'error') obj.error = obj.error.message + else scrub_errors(obj[key]) } } @@ -45,11 +72,11 @@ export async function load_expected_evaluation (name: string, exclude_full_path: return expected } -export async function load_actual_evaluation (name: string): Promise { +export async function load_actual_evaluation (evaluator: StoryEvaluator, name: string): Promise { const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`) const display_path = `${name}.yaml` const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml` - const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate() + const actual = await evaluator.evaluate({ display_path, full_path, story }) scrub_errors(actual) return actual }