diff --git a/tools/integ_tests/tester/StoryEvaluator.test.ts b/tools/integ_tests/tester/StoryEvaluator.test.ts new file mode 100644 index 000000000..0e7023069 --- /dev/null +++ b/tools/integ_tests/tester/StoryEvaluator.test.ts @@ -0,0 +1,55 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' +import { read_yaml } from '../../helpers' +import { type OpenAPIV3 } from 'openapi-types' + +describe('Story Evaluator', () => { + beforeAll(() => { + const spec = read_yaml('tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml') + create_shared_resources(spec as OpenAPIV3.Document) + }) + + test('passed', async () => { + const actual = await load_actual_evaluation('passed') + const expected = await load_expected_evaluation('passed') + expect(actual).toEqual(expected) + }) + + test('skipped', async () => { + const actual = await load_actual_evaluation('skipped') + const expected = await load_expected_evaluation('skipped') + expect(actual).toEqual(expected) + }) + + test('failed/not_found', async () => { + const actual = await load_actual_evaluation('failed/not_found') + const expected = await load_expected_evaluation('failed/not_found') + expect(actual).toEqual(expected) + }) + + test('failed/invalid_data', async () => { + const actual = await load_actual_evaluation('failed/invalid_data') + const expected = await load_expected_evaluation('failed/invalid_data') + expect(actual).toEqual(expected) + }) + + test('error/prologue_error', async () => { + const actual = await load_actual_evaluation('error/prologue_error') + const expected = await load_expected_evaluation('error/prologue_error') + expect(actual).toEqual(expected) + }) + + test('error/chapter_error', async () => { + const actual = await load_actual_evaluation('error/chapter_error') + const expected = await load_expected_evaluation('error/chapter_error') + expect(actual).toEqual(expected) + }) +}) \ No newline at end of file diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/integ_tests/tester/TestsRunner.test.ts similarity index 88% rename from tools/tests/tester/TestsRunner.test.ts rename to tools/integ_tests/tester/TestsRunner.test.ts index d2290d808..61af715aa 100644 --- a/tools/tests/tester/TestsRunner.test.ts +++ b/tools/integ_tests/tester/TestsRunner.test.ts @@ -15,8 +15,10 @@ import { load_expected_evaluation, scrub_errors } from './helpers' test('stories folder', async () => { // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') - const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {}) + const spec = read_yaml('tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml') + const runner = new TestsRunner( + spec as OpenAPIV3.Document, + 'tools/integ_tests/tester/fixtures/stories', {}) const actual_evaluations = await runner.run(true) as any[] for (const evaluation of actual_evaluations) scrub_errors(evaluation) for (const evaluation of actual_evaluations) { diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/integ_tests/tester/fixtures/evals/error/chapter_error.yaml similarity index 92% rename from tools/tests/tester/fixtures/evals/error/chapter_error.yaml rename to tools/integ_tests/tester/fixtures/evals/error/chapter_error.yaml index 5a3b6977c..7f7883429 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/integ_tests/tester/fixtures/evals/error/chapter_error.yaml @@ -1,5 +1,5 @@ display_path: error/chapter_error.yaml -full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml +full_path: tools/integ_tests/tester/fixtures/stories/error/chapter_error.yaml result: ERROR description: This story should failed due to missing info in the spec. diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/integ_tests/tester/fixtures/evals/error/prologue_error.yaml similarity index 88% rename from tools/tests/tester/fixtures/evals/error/prologue_error.yaml rename to tools/integ_tests/tester/fixtures/evals/error/prologue_error.yaml index 299fa9fee..65006c626 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/integ_tests/tester/fixtures/evals/error/prologue_error.yaml @@ -1,5 +1,5 @@ display_path: error/prologue_error.yaml -full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml +full_path: tools/integ_tests/tester/fixtures/stories/error/prologue_error.yaml result: ERROR description: This story should failed due to missing info in the spec. diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/integ_tests/tester/fixtures/evals/failed/invalid_data.yaml similarity index 94% rename from tools/tests/tester/fixtures/evals/failed/invalid_data.yaml rename to tools/integ_tests/tester/fixtures/evals/failed/invalid_data.yaml index 1cc11180a..6dd625fe4 100644 --- a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml +++ b/tools/integ_tests/tester/fixtures/evals/failed/invalid_data.yaml @@ -1,5 +1,5 @@ display_path: failed/invalid_data.yaml -full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml +full_path: tools/integ_tests/tester/fixtures/stories/failed/invalid_data.yaml result: FAILED description: This story should failed due invalid data. diff --git a/tools/tests/tester/fixtures/evals/failed/not_found.yaml b/tools/integ_tests/tester/fixtures/evals/failed/not_found.yaml similarity index 95% rename from tools/tests/tester/fixtures/evals/failed/not_found.yaml rename to tools/integ_tests/tester/fixtures/evals/failed/not_found.yaml index 06d186c73..df1e6d4b7 100644 --- a/tools/tests/tester/fixtures/evals/failed/not_found.yaml +++ b/tools/integ_tests/tester/fixtures/evals/failed/not_found.yaml @@ -1,5 +1,5 @@ display_path: failed/not_found.yaml -full_path: tools/tests/tester/fixtures/stories/failed/not_found.yaml +full_path: tools/integ_tests/tester/fixtures/stories/failed/not_found.yaml result: FAILED description: This story should failed due to missing info in the spec. diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/integ_tests/tester/fixtures/evals/passed.yaml similarity index 87% rename from tools/tests/tester/fixtures/evals/passed.yaml rename to tools/integ_tests/tester/fixtures/evals/passed.yaml index e7ec69ba3..8ed4fd9f7 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/integ_tests/tester/fixtures/evals/passed.yaml @@ -1,5 +1,5 @@ display_path: passed.yaml -full_path: tools/tests/tester/fixtures/stories/passed.yaml +full_path: tools/integ_tests/tester/fixtures/stories/passed.yaml result: PASSED description: This story should pass. diff --git a/tools/tests/tester/fixtures/evals/skipped.yaml b/tools/integ_tests/tester/fixtures/evals/skipped.yaml similarity index 59% rename from tools/tests/tester/fixtures/evals/skipped.yaml rename to tools/integ_tests/tester/fixtures/evals/skipped.yaml index 663cb1acf..da9035807 100644 --- a/tools/tests/tester/fixtures/evals/skipped.yaml +++ b/tools/integ_tests/tester/fixtures/evals/skipped.yaml @@ -1,5 +1,5 @@ display_path: skipped.yaml -full_path: "tools/tests/tester/fixtures/stories/skipped.yaml" +full_path: "tools/integ_tests/tester/fixtures/stories/skipped.yaml" result: SKIPPED description: This story should be skipped. diff --git a/tools/tests/tester/fixtures/specs/indices_excerpt.yaml b/tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml similarity index 100% rename from tools/tests/tester/fixtures/specs/indices_excerpt.yaml rename to tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml diff --git a/tools/tests/tester/fixtures/stories/error/chapter_error.yaml b/tools/integ_tests/tester/fixtures/stories/error/chapter_error.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/error/chapter_error.yaml rename to tools/integ_tests/tester/fixtures/stories/error/chapter_error.yaml diff --git a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml b/tools/integ_tests/tester/fixtures/stories/error/prologue_error.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/error/prologue_error.yaml rename to tools/integ_tests/tester/fixtures/stories/error/prologue_error.yaml diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/integ_tests/tester/fixtures/stories/failed/invalid_data.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/failed/invalid_data.yaml rename to tools/integ_tests/tester/fixtures/stories/failed/invalid_data.yaml diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/integ_tests/tester/fixtures/stories/failed/not_found.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/failed/not_found.yaml rename to tools/integ_tests/tester/fixtures/stories/failed/not_found.yaml diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/integ_tests/tester/fixtures/stories/passed.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/passed.yaml rename to tools/integ_tests/tester/fixtures/stories/passed.yaml diff --git a/tools/tests/tester/fixtures/stories/skipped.yaml b/tools/integ_tests/tester/fixtures/stories/skipped.yaml similarity index 100% rename from tools/tests/tester/fixtures/stories/skipped.yaml rename to tools/integ_tests/tester/fixtures/stories/skipped.yaml diff --git a/tools/tests/tester/helpers.ts b/tools/integ_tests/tester/helpers.ts similarity index 87% rename from tools/tests/tester/helpers.ts rename to tools/integ_tests/tester/helpers.ts index 99e274e48..d030d6980 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/integ_tests/tester/helpers.ts @@ -34,21 +34,21 @@ export function print_yaml (obj: any): void { export function scrub_errors (obj: any): void { for (const key in obj) { // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - if (key === 'error') obj[key] = obj[key].message + if (key === 'error') obj[key] = obj[key]?.message else if (typeof obj[key] === 'object') scrub_errors(obj[key]) } } export async function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Promise { - const expected = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`) + const expected = read_yaml(`tools/integ_tests/tester/fixtures/evals/${name}.yaml`) if (exclude_full_path) delete expected.full_path return expected } export async function load_actual_evaluation (name: string): Promise { - const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`) + const story: Story = read_yaml(`tools/integ_tests/tester/fixtures/stories/${name}.yaml`) const display_path = `${name}.yaml` - const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml` + const full_path = `tools/integ_tests/tester/fixtures/stories/${name}.yaml` const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate() scrub_errors(actual) return actual diff --git a/tools/tests/tester/overall_result.test.ts b/tools/integ_tests/tester/overall_result.test.ts similarity index 100% rename from tools/tests/tester/overall_result.test.ts rename to tools/integ_tests/tester/overall_result.test.ts diff --git a/tools/src/tester/start.ts b/tools/src/tester/start.ts index 797602158..995db223f 100644 --- a/tools/src/tester/start.ts +++ b/tools/src/tester/start.ts @@ -32,6 +32,7 @@ const options = { // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge() const runner = new TestsRunner(spec, opts.testsPath, options) void runner.run().then(() => { _.noop() }) diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts deleted file mode 100644 index 3cc57aa90..000000000 --- a/tools/tests/tester/StoryEvaluator.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' -import { read_yaml } from '../../helpers' -import { type OpenAPIV3 } from 'openapi-types' - -const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') -create_shared_resources(spec as OpenAPIV3.Document) - -test('passed', async () => { - const actual = await load_actual_evaluation('passed') - const expected = await load_expected_evaluation('passed') - expect(actual).toEqual(expected) -}) - -test('skipped', async () => { - const actual = await load_actual_evaluation('skipped') - const expected = await load_expected_evaluation('skipped') - expect(actual).toEqual(expected) -}) - -test('failed/not_found', async () => { - const actual = await load_actual_evaluation('failed/not_found') - const expected = await load_expected_evaluation('failed/not_found') - expect(actual).toEqual(expected) -}) - -test('failed/invalid_data', async () => { - const actual = await load_actual_evaluation('failed/invalid_data') - const expected = await load_expected_evaluation('failed/invalid_data') - expect(actual).toEqual(expected) -}) - -test('error/prologue_error', async () => { - const actual = await load_actual_evaluation('error/prologue_error') - const expected = await load_expected_evaluation('error/prologue_error') - expect(actual).toEqual(expected) -}) - -test('error/chapter_error', async () => { - const actual = await load_actual_evaluation('error/chapter_error') - const expected = await load_expected_evaluation('error/chapter_error') - expect(actual).toEqual(expected) -})