Split unit and integration tests.

tools/integ_tests/tester/StoryEvaluator.test.ts

@@ -0,0 +1,55 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
+import { read_yaml } from '../../helpers'
+import { type OpenAPIV3 } from 'openapi-types'
+
+describe('Story Evaluator', () => {
+  beforeAll(() => {
+    const spec = read_yaml('tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml')
+    create_shared_resources(spec as OpenAPIV3.Document)  
+  })
+
+  test('passed', async () => {
+    const actual = await load_actual_evaluation('passed')
+    const expected = await load_expected_evaluation('passed')
+    expect(actual).toEqual(expected)
+  })
+
+  test('skipped', async () => {
+    const actual = await load_actual_evaluation('skipped')
+    const expected = await load_expected_evaluation('skipped')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/not_found', async () => {
+    const actual = await load_actual_evaluation('failed/not_found')
+    const expected = await load_expected_evaluation('failed/not_found')
+    expect(actual).toEqual(expected)
+  })
+
+  test('failed/invalid_data', async () => {
+    const actual = await load_actual_evaluation('failed/invalid_data')
+    const expected = await load_expected_evaluation('failed/invalid_data')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/prologue_error', async () => {
+    const actual = await load_actual_evaluation('error/prologue_error')
+    const expected = await load_expected_evaluation('error/prologue_error')
+    expect(actual).toEqual(expected)
+  })
+
+  test('error/chapter_error', async () => {
+    const actual = await load_actual_evaluation('error/chapter_error')
+    const expected = await load_expected_evaluation('error/chapter_error')
+    expect(actual).toEqual(expected)
+  })
+})

tools/tests/tester/TestsRunner.test.ts → tools/integ_tests/tester/TestsRunner.test.ts

@@ -15,8 +15,10 @@ import { load_expected_evaluation, scrub_errors } from './helpers'
 test('stories folder', async () => {
   // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
   process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
-  const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
-  const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
+  const spec = read_yaml('tools/integ_tests/tester/fixtures/specs/indices_excerpt.yaml')
+  const runner = new TestsRunner(
+    spec as OpenAPIV3.Document,
+    'tools/integ_tests/tester/fixtures/stories', {})
   const actual_evaluations = await runner.run(true) as any[]
   for (const evaluation of actual_evaluations) scrub_errors(evaluation)
   for (const evaluation of actual_evaluations) {

tools/tests/tester/fixtures/evals/error/chapter_error.yaml → tools/integ_tests/tester/fixtures/evals/error/chapter_error.yaml

@@ -1,5 +1,5 @@
 display_path: error/chapter_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml
+full_path: tools/integ_tests/tester/fixtures/stories/error/chapter_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/error/prologue_error.yaml → tools/integ_tests/tester/fixtures/evals/error/prologue_error.yaml

@@ -1,5 +1,5 @@
 display_path: error/prologue_error.yaml
-full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
+full_path: tools/integ_tests/tester/fixtures/stories/error/prologue_error.yaml
 
 result: ERROR
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/failed/invalid_data.yaml → tools/integ_tests/tester/fixtures/evals/failed/invalid_data.yaml

@@ -1,5 +1,5 @@
 display_path: failed/invalid_data.yaml
-full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
+full_path: tools/integ_tests/tester/fixtures/stories/failed/invalid_data.yaml
 
 result: FAILED
 description: This story should failed due invalid data.

tools/tests/tester/fixtures/evals/failed/not_found.yaml → tools/integ_tests/tester/fixtures/evals/failed/not_found.yaml

@@ -1,5 +1,5 @@
 display_path: failed/not_found.yaml
-full_path: tools/tests/tester/fixtures/stories/failed/not_found.yaml
+full_path: tools/integ_tests/tester/fixtures/stories/failed/not_found.yaml
 
 result: FAILED
 description: This story should failed due to missing info in the spec.

tools/tests/tester/fixtures/evals/passed.yaml → tools/integ_tests/tester/fixtures/evals/passed.yaml

@@ -1,5 +1,5 @@
 display_path: passed.yaml
-full_path: tools/tests/tester/fixtures/stories/passed.yaml
+full_path: tools/integ_tests/tester/fixtures/stories/passed.yaml
 
 result: PASSED
 description: This story should pass.

tools/tests/tester/fixtures/evals/skipped.yaml → tools/integ_tests/tester/fixtures/evals/skipped.yaml

@@ -1,5 +1,5 @@
 display_path: skipped.yaml
-full_path: "tools/tests/tester/fixtures/stories/skipped.yaml"
+full_path: "tools/integ_tests/tester/fixtures/stories/skipped.yaml"
 
 result: SKIPPED
 description: This story should be skipped.

tools/tests/tester/helpers.ts → tools/integ_tests/tester/helpers.ts

@@ -34,21 +34,21 @@ export function print_yaml (obj: any): void {
 export function scrub_errors (obj: any): void {
   for (const key in obj) {
     // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
-    if (key === 'error') obj[key] = obj[key].message
+    if (key === 'error') obj[key] = obj[key]?.message
     else if (typeof obj[key] === 'object') scrub_errors(obj[key])
   }
 }
 
 export async function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Promise<any> {
-  const expected = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`)
+  const expected = read_yaml(`tools/integ_tests/tester/fixtures/evals/${name}.yaml`)
   if (exclude_full_path) delete expected.full_path
   return expected
 }
 
 export async function load_actual_evaluation (name: string): Promise<StoryEvaluation> {
-  const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`)
+  const story: Story = read_yaml(`tools/integ_tests/tester/fixtures/stories/${name}.yaml`)
   const display_path = `${name}.yaml`
-  const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml`
+  const full_path = `tools/integ_tests/tester/fixtures/stories/${name}.yaml`
   const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate()
   scrub_errors(actual)
   return actual

tools/src/tester/start.ts

@@ -32,6 +32,7 @@ const options = {
 
 // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml
 process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!'
+
 const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge()
 const runner = new TestsRunner(spec, opts.testsPath, options)
 void runner.run().then(() => { _.noop() })