diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml
index eb6aa73e9..46738ade5 100644
--- a/.github/opensearch-cluster/docker-compose.yml
+++ b/.github/opensearch-cluster/docker-compose.yml
@@ -8,4 +8,4 @@ services:
       - "9600:9600"
     environment:
       - "discovery.type=single-node"
-      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD}"
\ No newline at end of file
+      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}"
\ No newline at end of file
diff --git a/.github/workflows/test-spec.yaml b/.github/workflows/test-spec.yml
similarity index 100%
rename from .github/workflows/test-spec.yaml
rename to .github/workflows/test-spec.yml
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index f261c5e86..550dc7a38 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -145,6 +145,64 @@ This repository includes several OpenAPI Specification Extensions to fill in any
 - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml).
 - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema.
 
+## Adding tests for the spec
+
+To assure the correctness of the spec, you must add tests for the spec in the [tests/](tests) directory. Each yaml file in the tests directory represents a test story that tests a collection of related operations. A test story has 3 main components:
+- prologues: These are the operations that are executed before the test story is run. They are used to set up the environment for the test story.
+- chapters: These are the operations that are being tested.
+- epilogues: These are the operations that are executed after the test story is run. They are used to clean up the environment after the test story.
+
+Below is an example test story that tests the index operations:
+```yaml
+$schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly.
+
+skip: false # Skip this test story if set to true.
+description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion.
+
+prologues: [] # No prologues are needed for this story.
+
+epilogues: # Clean up the environment by assuring that the `books` index is deleted afterward.
+  - path: /books
+    method: DELETE
+    status: [200, 404] # The index may not exist, so we accept 404 as a valid response.
+    
+chapters:
+  - synopsis: Create an index named `books` with mappings and settings.
+    path: /{index} # The test will fail if "PUT /{index}" operation is not found in the spec.
+    method: PUT
+    parameters: # All parameters are validated against their schemas in the spec
+      index: books
+    request_body: # The request body is validated against the schema of the requestBody in the spec
+      payload:
+        mappings:
+          properties:
+            name:
+              type: keyword
+            age:
+              type: integer
+        settings:
+          number_of_shards: 5
+          number_of_replicas: 2
+    response: # The response body is validated against the schema of the corresponding response in the spec
+      status: 200 # This is the expected status code of the response. Any other status code will fail the test.
+
+  - synopsis: Retrieve the mappings and settings of the `books` index.
+    path: /{index}
+    method: GET
+    parameters:
+      index: books
+      flat_settings: true
+      
+  - synopsis: Delete the `books` index.
+    path: /{index}
+    method: DELETE
+    parameters:
+      index: books
+```
+
+Check the [test_story JSON Schema](json_schemas/test_story.schema.yaml) for the complete structure of a test story.
+
+
 ## Tools
 
 A number of [tools](tools) have been authored using TypeScript to aid in the development of the specification. These largely center around linting and merging the multi-file spec layout.
diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts
index 03b3c6497..3f20b0eda 100644
--- a/tools/src/tester/ChapterEvaluator.ts
+++ b/tools/src/tester/ChapterEvaluator.ts
@@ -21,10 +21,11 @@ export default class ChapterEvaluator {
     this.schema_validator = SharedResources.get_instance().schema_validator
   }
 
-  async evaluate (skipped: boolean): Promise<ChapterEvaluation> {
-    if (skipped) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
-    const operation = this.spec_parser.locate_operation(this.chapter)
+  async evaluate (skip: boolean): Promise<ChapterEvaluation> {
+    if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } }
     const response = await this.chapter_reader.read(this.chapter)
+    const operation = this.spec_parser.locate_operation(this.chapter)
+    if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } }
     const params = this.#evaluate_parameters(operation)
     const request_body = this.#evaluate_request_body(operation)
     const status = this.#evaluate_status(response)
@@ -32,7 +33,7 @@ export default class ChapterEvaluator {
     return {
       title: this.chapter.synopsis,
       overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) },
-      request: { parameters: params, requestBody: request_body },
+      request: { parameters: params, request_body },
       response: { status, payload }
     }
   }
diff --git a/tools/src/tester/ResultsDisplayer.ts b/tools/src/tester/ResultsDisplayer.ts
index 7a0d04f47..51678a910 100644
--- a/tools/src/tester/ResultsDisplayer.ts
+++ b/tools/src/tester/ResultsDisplayer.ts
@@ -48,7 +48,7 @@ export default class ResultsDisplayer {
     if (chapter.overall.result === Result.PASSED || chapter.overall.result === Result.SKIPPED) return
 
     this.#display_parameters(chapter.request?.parameters ?? {})
-    this.#display_request_body(chapter.request?.requestBody)
+    this.#display_request_body(chapter.request?.request_body)
     this.#display_status(chapter.response?.status)
     this.#display_payload(chapter.response?.payload)
   }
diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts
index 4f9c91896..af5ec1d46 100644
--- a/tools/src/tester/SchemaValidator.ts
+++ b/tools/src/tester/SchemaValidator.ts
@@ -6,7 +6,7 @@ import { type Evaluation, Result } from './types/eval.types'
 export default class SchemaValidator {
   private readonly ajv: AJV
   constructor (spec: OpenAPIV3.Document) {
-    this.ajv = new AJV()
+    this.ajv = new AJV({ allErrors: true, strict: true })
     addFormats(this.ajv)
     this.ajv.addKeyword('discriminator')
     const schemas = spec.components?.schemas ?? {}
diff --git a/tools/src/tester/SpecParser.ts b/tools/src/tester/SpecParser.ts
index ecc171f8e..c2af92d4f 100644
--- a/tools/src/tester/SpecParser.ts
+++ b/tools/src/tester/SpecParser.ts
@@ -12,13 +12,13 @@ export default class SpecParser {
     this.spec = spec
   }
 
-  locate_operation (chapter: Chapter): ParsedOperation {
+  locate_operation (chapter: Chapter): ParsedOperation | undefined {
     const path = chapter.path
     const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
     const cache_key = path + method
     if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
     const operation = this.spec.paths[path]?.[method]
-    if (operation == null) throw new Error(`Operation "${method.toUpperCase()} ${path}" not found in the spec.`)
+    if (operation == null) return undefined
     this.#deref(operation)
     const parameters = _.keyBy(operation.parameters ?? [], 'name')
     // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
index 7c049125c..f47d06523 100644
--- a/tools/src/tester/StoryEvaluator.ts
+++ b/tools/src/tester/StoryEvaluator.ts
@@ -50,18 +50,13 @@ export default class StoryEvaluator {
   }
 
   async #evaluate_chapters (chapters: Chapter[]): Promise<ChapterEvaluation[]> {
-    if (this.has_errors) return []
-    let has_errors: boolean = this.has_errors
-
     const evaluations: ChapterEvaluation[] = []
-
     for (const chapter of chapters) {
       const evaluator = new ChapterEvaluator(chapter)
-      const evaluation = await evaluator.evaluate(has_errors)
-      has_errors = has_errors || evaluation.overall.result === Result.ERROR
+      const evaluation = await evaluator.evaluate(this.has_errors)
+      this.has_errors = this.has_errors || evaluation.overall.result === Result.ERROR
       evaluations.push(evaluation)
     }
-
     return evaluations
   }
 
@@ -70,7 +65,7 @@ export default class StoryEvaluator {
     for (const chapter of chapters) {
       const title = `${chapter.method} ${chapter.path}`
       const response = await this.chapter_reader.read(chapter)
-      const status = chapter.status ?? []
+      const status = chapter.status ?? [200, 201]
       if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } })
       else {
         this.has_errors = true
diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts
index a5ba0689f..a90e344d7 100644
--- a/tools/src/tester/TestsRunner.ts
+++ b/tools/src/tester/TestsRunner.ts
@@ -6,7 +6,7 @@ import StoryEvaluator, { type StoryFile } from './StoryEvaluator'
 import fs from 'fs'
 import { type Story } from './types/story.types'
 import { read_yaml } from '../../helpers'
-import { Result } from './types/eval.types'
+import { Result, type StoryEvaluation } from './types/eval.types'
 import ResultsDisplayer, { type DisplayOptions } from './ResultsDisplayer'
 import SharedResources from './SharedResources'
 import { resolve, basename } from 'path'
@@ -27,17 +27,20 @@ export default class TestsRunner {
     SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser })
   }
 
-  async run (): Promise<void> {
+  async run (debug: boolean = false): Promise<StoryEvaluation[]> {
     let failed = false
-    const story_files = this.#collect_story_files(this.path, '', '').sort((a, b) => a.display_path.localeCompare(b.display_path))
-    for (const story_file of story_files) {
+    const story_files = this.#collect_story_files(this.path, '', '')
+    const evaluations: StoryEvaluation[] = []
+    for (const story_file of this.#sort_story_files(story_files)) {
       const evaluator = new StoryEvaluator(story_file)
       const evaluation = await evaluator.evaluate()
       const displayer = new ResultsDisplayer(evaluation, this.opts)
-      displayer.display()
+      if (debug) evaluations.push(evaluation)
+      else displayer.display()
       if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true
     }
-    if (failed) process.exit(1)
+    if (failed && !debug) process.exit(1)
+    return evaluations
   }
 
   #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] {
@@ -56,4 +59,13 @@ export default class TestsRunner {
       })
     }
   }
+
+  #sort_story_files (story_files: StoryFile[]): StoryFile[] {
+    return story_files.sort((a, b) => {
+      const a_depth = a.display_path.split('/').length
+      const b_depth = b.display_path.split('/').length
+      if (a_depth !== b_depth) return a_depth - b_depth
+      return a.display_path.localeCompare(b.display_path)
+    })
+  }
 }
diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts
index 95f9e7af7..a99238eec 100644
--- a/tools/src/tester/types/eval.types.ts
+++ b/tools/src/tester/types/eval.types.ts
@@ -16,7 +16,7 @@ export interface ChapterEvaluation {
   overall: Evaluation
   request?: {
     parameters?: Record<string, Evaluation>
-    requestBody?: Evaluation
+    request_body?: Evaluation
   }
   response?: {
     status: Evaluation
diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts
new file mode 100644
index 000000000..524be1d94
--- /dev/null
+++ b/tools/tests/tester/StoryEvaluator.test.ts
@@ -0,0 +1,42 @@
+import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers'
+import { read_yaml } from '../../helpers'
+import { type OpenAPIV3 } from 'openapi-types'
+
+const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+create_shared_resources(spec as OpenAPIV3.Document)
+
+test('passed', async () => {
+  const actual = await load_actual_evaluation('passed')
+  const expected = await load_expected_evaluation('passed')
+  expect(actual).toEqual(expected)
+})
+
+test('skipped', async () => {
+  const actual = await load_actual_evaluation('skipped')
+  const expected = await load_expected_evaluation('skipped')
+  expect(actual).toEqual(expected)
+})
+
+test('failed/not_found', async () => {
+  const actual = await load_actual_evaluation('failed/not_found')
+  const expected = await load_expected_evaluation('failed/not_found')
+  expect(actual).toEqual(expected)
+})
+
+test('failed/invalid_data', async () => {
+  const actual = await load_actual_evaluation('failed/invalid_data')
+  const expected = await load_expected_evaluation('failed/invalid_data')
+  expect(actual).toEqual(expected)
+})
+
+test('error/prologue_error', async () => {
+  const actual = await load_actual_evaluation('error/prologue_error')
+  const expected = await load_expected_evaluation('error/prologue_error')
+  expect(actual).toEqual(expected)
+})
+
+test('error/chapter_error', async () => {
+  const actual = await load_actual_evaluation('error/chapter_error')
+  const expected = await load_expected_evaluation('error/chapter_error')
+  expect(actual).toEqual(expected)
+})
diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/tests/tester/TestsRunner.test.ts
new file mode 100644
index 000000000..ef4a73892
--- /dev/null
+++ b/tools/tests/tester/TestsRunner.test.ts
@@ -0,0 +1,27 @@
+import { read_yaml } from '../../helpers'
+import TestsRunner from '../../src/tester/TestsRunner'
+import { type OpenAPIV3 } from 'openapi-types'
+import { load_expected_evaluation, scrub_errors } from './helpers'
+
+test('stories folder', async () => {
+  // The password must match the one specified in .github/workflows/test-spec.yml
+  process.env.OPENSEARCH_PASSWORD = 'myStrongPassword123!'
+  const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml')
+  const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {})
+  const actual_evaluations = await runner.run(true) as any[]
+  for (const evaluation of actual_evaluations) scrub_errors(evaluation)
+  for (const evaluation of actual_evaluations) {
+    expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy()
+    delete evaluation.full_path
+  }
+
+  const skipped = await load_expected_evaluation('skipped', true)
+  const passed = await load_expected_evaluation('passed', true)
+  const not_found = await load_expected_evaluation('failed/not_found', true)
+  const invalid_data = await load_expected_evaluation('failed/invalid_data', true)
+  const chapter_error = await load_expected_evaluation('error/chapter_error', true)
+  const prologue_error = await load_expected_evaluation('error/prologue_error', true)
+
+  const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found]
+  expect(actual_evaluations).toEqual(expected_evaluations)
+})
diff --git a/tools/tests/tester/ansi.test.ts b/tools/tests/tester/ansi.test.ts
index aa163444b..ef75924ff 100644
--- a/tools/tests/tester/ansi.test.ts
+++ b/tools/tests/tester/ansi.test.ts
@@ -1,17 +1,40 @@
 import * as ansi from '../../src/tester/Ansi'
 
-test('b', async () => {
+test('b', () => {
   expect(ansi.b('xyz')).toEqual('\x1b[1mxyz\x1b[0m')
 })
 
-test('i', async () => {
+test('i', () => {
   expect(ansi.i('xyz')).toEqual('\x1b[3mxyz\x1b[0m')
 })
 
-test.todo('padding')
-test.todo('green')
-test.todo('red')
-test.todo('yellow')
-test.todo('cyan')
-test.todo('gray')
-test.todo('magenta')
+test('padding', () => {
+  expect(ansi.padding('xyz', 10)).toEqual('xyz       ')
+  expect(ansi.padding('xyz', 10, 2)).toEqual('  xyz       ')
+  expect(ansi.padding('xyz', 10, 8)).toEqual('        xyz       ')
+  expect(ansi.padding('xyz', 2)).toEqual('xyz')
+})
+
+test('green', () => {
+  expect(ansi.green('xyz')).toEqual('\x1b[32mxyz\x1b[0m')
+})
+
+test('red', () => {
+  expect(ansi.red('xyz')).toEqual('\x1b[31mxyz\x1b[0m')
+})
+
+test('yellow', () => {
+  expect(ansi.yellow('xyz')).toEqual('\x1b[33mxyz\x1b[0m')
+})
+
+test('cyan', () => {
+  expect(ansi.cyan('xyz')).toEqual('\x1b[36mxyz\x1b[0m')
+})
+
+test('gray', () => {
+  expect(ansi.gray('xyz')).toEqual('\x1b[90mxyz\x1b[0m')
+})
+
+test('magenta', () => {
+  expect(ansi.magenta('xyz')).toEqual('\x1b[35mxyz\x1b[0m')
+})
diff --git a/tools/tests/tester/fixtures/empty.yaml b/tools/tests/tester/fixtures/empty.yaml
deleted file mode 100644
index 0b63cb9b6..000000000
--- a/tools/tests/tester/fixtures/empty.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-$schema: ../json_schemas/test_story.schema.yaml
-
-chapters: []
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml
new file mode 100644
index 000000000..5a3b6977c
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml
@@ -0,0 +1,39 @@
+display_path: error/chapter_error.yaml
+full_path: tools/tests/tester/fixtures/stories/error/chapter_error.yaml
+
+result: ERROR
+description: This story should failed due to missing info in the spec.
+
+prologues:
+  - title: PUT /books
+    overall:
+      result: PASSED
+
+chapters:
+  - title: This chapter should fail.
+    overall:
+      result: FAILED
+      message: Operation "GET /{index}/settings" not found in the spec.
+  - title: This chapter show throw an error.
+    overall:
+      result: ERROR
+    request:
+      parameters: {}
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: ERROR
+        message: 'Expected status 200, but received 404: application/json. no such index
+            [undefined]'
+        error: Request failed with status code 404
+      payload:
+        result: SKIPPED
+  - title: This chapter should be skipped.
+    overall:
+      result: SKIPPED
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
new file mode 100644
index 000000000..299fa9fee
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
@@ -0,0 +1,28 @@
+display_path: error/prologue_error.yaml
+full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml
+
+result: ERROR
+description: This story should failed due to missing info in the spec.
+
+prologues:
+  - title: PUT /books
+    overall:
+      result: PASSED
+  - title: DELETE /does_not_exists
+    overall:
+      result: ERROR
+      message: no such index [does_not_exists]
+      error: Request failed with status code 404
+
+chapters:
+  - title: This chapter be skipped.
+    overall:
+      result: SKIPPED
+  - title: This chapter be skipped.
+    overall:
+      result: SKIPPED
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml
new file mode 100644
index 000000000..1cc11180a
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml
@@ -0,0 +1,62 @@
+display_path: failed/invalid_data.yaml
+full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
+
+result: FAILED
+description: This story should failed due invalid data.
+
+prologues: []
+
+chapters:
+  - title: This chapter should fail because the parameter is invalid.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: FAILED
+          message: data must be string
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the request body is invalid.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: FAILED
+        message: data must NOT have additional properties
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the response is invalid.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: FAILED
+        message: data must NOT have additional properties
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED
+  - title: DELETE /30
+    overall:
+      result: PASSED
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/failed/not_found.yaml b/tools/tests/tester/fixtures/evals/failed/not_found.yaml
new file mode 100644
index 000000000..06d186c73
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/failed/not_found.yaml
@@ -0,0 +1,65 @@
+display_path: failed/not_found.yaml
+full_path: tools/tests/tester/fixtures/stories/failed/not_found.yaml
+
+result: FAILED
+description: This story should failed due to missing info in the spec.
+
+prologues: []
+
+chapters:
+  - title: This chapter should fail because the operation is not defined in the spec.
+    overall:
+      result: FAILED
+      message: Operation "GET /_cat/health" not found in the spec.
+  - title: This chapter should fail because the parameter is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+        timeout:
+          result: FAILED
+          message: Schema for "timeout" parameter not found.
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the request body is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: FAILED
+        message: Schema for "application/json" request body not found in the spec.
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+  - title: This chapter should fail because the response is not defined in the spec.
+    overall:
+      result: FAILED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: FAILED
+        message: 'Schema for "404: application/json" response not found in the spec.'
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml
new file mode 100644
index 000000000..e7ec69ba3
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/passed.yaml
@@ -0,0 +1,28 @@
+display_path: passed.yaml
+full_path: tools/tests/tester/fixtures/stories/passed.yaml
+
+result: PASSED
+description: This story should pass.
+
+prologues: []
+
+chapters:
+  - title: This chapter should pass.
+    overall:
+      result: PASSED
+    request:
+      parameters:
+        index:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload:
+        result: PASSED
+
+epilogues:
+  - title: DELETE /books
+    overall:
+      result: PASSED
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/evals/skipped.yaml b/tools/tests/tester/fixtures/evals/skipped.yaml
new file mode 100644
index 000000000..663cb1acf
--- /dev/null
+++ b/tools/tests/tester/fixtures/evals/skipped.yaml
@@ -0,0 +1,7 @@
+display_path: skipped.yaml
+full_path: "tools/tests/tester/fixtures/stories/skipped.yaml"
+
+result: SKIPPED
+description: This story should be skipped.
+
+chapters: []
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/specs/indices_excerpt.yaml b/tools/tests/tester/fixtures/specs/indices_excerpt.yaml
new file mode 100644
index 000000000..b495172eb
--- /dev/null
+++ b/tools/tests/tester/fixtures/specs/indices_excerpt.yaml
@@ -0,0 +1,177 @@
+openapi: 3.1.0
+info:
+  title: Test API
+  version: 1.0.0
+paths:
+  /{index}:
+    delete:
+      operationId: indices.delete.0
+      x-operation-group: indices.delete
+      x-version-added: '1.0'
+      description: Deletes an index.
+      externalDocs:
+        url: https://opensearch.org/docs/latest/api-reference/index-apis/delete-index/
+      parameters:
+        - $ref: '#/components/parameters/indices.delete::path.index'
+        - $ref: '#/components/parameters/indices.delete::query.timeout'
+      responses:
+        '200':
+          $ref: '#/components/responses/indices.delete@200'
+    head:
+      operationId: indices.exists.0
+      x-operation-group: indices.exists
+      x-version-added: '1.0'
+      description: Returns information about whether a particular index exists.
+      externalDocs:
+        url: https://opensearch.org/docs/latest/api-reference/index-apis/exists/
+      parameters:
+        - $ref: '#/components/parameters/indices.exists::path.index'
+      responses:
+        '200':
+          $ref: '#/components/responses/indices.exists@200'
+    put:
+      operationId: indices.create.0
+      x-operation-group: indices.create
+      x-version-added: '1.0'
+      description: Creates an index with optional settings and mappings.
+      externalDocs:
+        url: https://opensearch.org/docs/latest/api-reference/index-apis/create-index/
+      parameters:
+        - $ref: '#/components/parameters/indices.create::path.index'
+      requestBody:
+        $ref: '#/components/requestBodies/indices.create'
+      responses:
+        '200':
+          $ref: '#/components/responses/indices.create@200'
+components:
+  requestBodies:
+    indices.create:
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              settings:
+                type: object
+                description: Optional settings for the index.
+              mappings:
+                type: object
+                description: Optional mappings for the index.
+            additionalProperties: false
+  responses:
+    indices.delete@200:
+      description: ''
+      content:
+        application/json:
+          schema:
+            $ref: '#/components/schemas/_common:IndicesResponseBase'
+    indices.exists@200:
+      description: ''
+      content:
+        application/json: {}
+    indices.create@200:
+      description: ''
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              index:
+                $ref: '#/components/schemas/_common:IndexName'
+              shards_acknowledged:
+                type: boolean
+              acknowledged:
+                type: boolean
+            required:
+              - index
+              - shards_acknowledged
+              - acknowledged
+  parameters:
+    indices.delete::path.index:
+      in: path
+      name: index
+      description: |-
+        Comma-separated list of indices to delete.
+        You cannot specify index aliases.
+        By default, this parameter does not support wildcards (`*`) or `_all`.
+        To use wildcards or `_all`, set the `action.destructive_requires_name` cluster setting to `false`.
+      required: true
+      schema:
+        $ref: '#/components/schemas/_common:Indices'
+    indices.delete::query.timeout:
+      in: query
+      name: timeout
+      description: |-
+        Period to wait for a response.
+        If no response is received before the timeout expires, the request fails and returns an error.
+      schema:
+        $ref: '#/components/schemas/_common:Duration'
+      style: form
+    indices.exists::path.index:
+      in: path
+      name: index
+      description: Comma-separated list of data streams, indices, and aliases. Supports wildcards (`*`).
+      required: true
+      schema:
+        $ref: '#/components/schemas/_common:Indices'
+      style: simple
+    indices.create::path.index:
+      in: path
+      name: index
+      description: Name of the index you wish to create.
+      required: true
+      schema:
+        $ref: '#/components/schemas/_common:IndexName'
+      style: simple
+  schemas:
+    _common:uint:
+      type: number
+    _common:ShardStatistics:
+      type: object
+      properties:
+        failed:
+          $ref: '#/components/schemas/_common:uint'
+        successful:
+          $ref: '#/components/schemas/_common:uint'
+        total:
+          $ref: '#/components/schemas/_common:uint'
+        failures:
+          type: array
+          items:
+            type: object
+        skipped:
+          $ref: '#/components/schemas/_common:uint'
+      required:
+        - failed
+        - successful
+        - total
+    _common:AcknowledgedResponseBase:
+      type: object
+      properties:
+        acknowledged:
+          description: For a successful response, this value is always true. On failure, an exception is returned instead.
+          type: boolean
+      required:
+        - acknowledged
+    _common:IndicesResponseBase:
+      allOf:
+        - $ref: '#/components/schemas/_common:AcknowledgedResponseBase'
+        - type: object
+          properties:
+            _shards:
+              $ref: '#/components/schemas/_common:ShardStatistics'
+          additionalProperties: false
+    _common:Indices:
+      oneOf:
+        - $ref: '#/components/schemas/_common:IndexName'
+        - type: array
+          items:
+            $ref: '#/components/schemas/_common:IndexName'
+    _common:IndexName:
+      type: string
+    _common:Duration:
+      description: |-
+        A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and
+        `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value.
+      pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$
+      type: string
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/stories/error/chapter_error.yaml b/tools/tests/tester/fixtures/stories/error/chapter_error.yaml
new file mode 100644
index 000000000..2272a666d
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/error/chapter_error.yaml
@@ -0,0 +1,20 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should failed due to missing info in the spec.
+prologues:
+  - path: /books
+    method: PUT
+epilogues:
+  - path: /books
+    method: DELETE
+chapters:
+  - synopsis: This chapter should fail.
+    path: /{index}/settings
+    method: GET
+  - synopsis: This chapter show throw an error.
+    path: /{index}
+    method: DELETE
+  - synopsis: This chapter should be skipped.
+    path: /_cat/indices
+    method: GET
+
diff --git a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml
new file mode 100644
index 000000000..8b11deb92
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml
@@ -0,0 +1,19 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should failed due to missing info in the spec.
+prologues:
+  - path: /books
+    method: PUT
+  - path: /does_not_exists
+    method: DELETE
+epilogues:
+  - path: /books
+    method: DELETE
+chapters:
+  - synopsis: This chapter be skipped.
+    path: /_cat/health
+    method: GET
+  - synopsis: This chapter be skipped.
+    path: /_cat/indices
+    method: GET
+
diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
new file mode 100644
index 000000000..fbb6573b2
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml
@@ -0,0 +1,33 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should failed due invalid data.
+epilogues:
+  - path: /books
+    method: DELETE
+    status: [200, 404]
+  - path: /30
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: This chapter should fail because the parameter is invalid.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: 30
+  - synopsis: This chapter should fail because the request body is invalid.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: books
+    request_body:
+      payload:
+        aliases: {}
+  - synopsis: This chapter should fail because the response is invalid.
+    path: /{index}
+    method: DELETE
+    parameters:
+      index: books
+    response:
+      status: 200
+      payload:
+        shards_acknowledged: true
\ No newline at end of file
diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/failed/not_found.yaml
new file mode 100644
index 000000000..b210f74e9
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/failed/not_found.yaml
@@ -0,0 +1,31 @@
+$schema: ../../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should failed due to missing info in the spec.
+epilogues:
+  - path: /books
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: This chapter should fail because the operation is not defined in the spec.
+    path: /_cat/health
+    method: GET
+  - synopsis: This chapter should fail because the parameter is not defined in the spec.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: books
+      timeout: 30s
+  - synopsis: This chapter should fail because the request body is not defined in the spec.
+    path: /{index}
+    method: HEAD
+    parameters:
+      index: books
+    request_body:
+      payload: {}
+  - synopsis: This chapter should fail because the response is not defined in the spec.
+    path: /{index}
+    method: DELETE
+    parameters:
+      index: movies
+    response:
+      status: 404
diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml
new file mode 100644
index 000000000..4a7fbfea3
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/passed.yaml
@@ -0,0 +1,13 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should pass.
+epilogues:
+  - path: /books
+    method: DELETE
+    status: [200, 404]
+chapters:
+  - synopsis: This chapter should pass.
+    path: /{index}
+    method: PUT
+    parameters:
+      index: books
diff --git a/tools/tests/tester/fixtures/stories/skipped.yaml b/tools/tests/tester/fixtures/stories/skipped.yaml
new file mode 100644
index 000000000..256d65094
--- /dev/null
+++ b/tools/tests/tester/fixtures/stories/skipped.yaml
@@ -0,0 +1,17 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+skip: true
+description: This story should be skipped.
+prologues:
+  - path: /_cluster/settings
+    method: GET
+epilogues:
+  - path: /_cluster/settings
+    method: PUT
+chapters:
+  - synopsis: This chapter should not be executed.
+    path: /_cat/health
+    method: GET
+  - synopsis: This chapter should not be executed.
+    path: /_cat/indices
+    method: GET
\ No newline at end of file
diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts
new file mode 100644
index 000000000..49b47bf39
--- /dev/null
+++ b/tools/tests/tester/helpers.ts
@@ -0,0 +1,46 @@
+import ChapterReader from '../../src/tester/ChapterReader'
+import SpecParser from '../../src/tester/SpecParser'
+import SchemaValidator from '../../src/tester/SchemaValidator'
+import SharedResources from '../../src/tester/SharedResources'
+import { type OpenAPIV3 } from 'openapi-types'
+import YAML from 'yaml'
+import type { StoryEvaluation } from '../../src/tester/types/eval.types'
+import type { Story } from '../../src/tester/types/story.types'
+import { read_yaml } from '../../helpers'
+import StoryEvaluator from '../../src/tester/StoryEvaluator'
+
+export function create_shared_resources (spec: any): void {
+  // The password must match the one specified in .github/workflows/test-spec.yml
+  process.env.OPENSEARCH_PASSWORD = 'myStrongPassword123!'
+  const chapter_reader = new ChapterReader()
+  const spec_parser = new SpecParser(spec as OpenAPIV3.Document)
+  const schema_validator = new SchemaValidator(spec as OpenAPIV3.Document)
+  SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser })
+}
+
+export function print_yaml (obj: any): void {
+  console.log(YAML.stringify(obj, { indent: 2, singleQuote: true, lineWidth: undefined }))
+}
+
+export function scrub_errors (obj: any): void {
+  for (const key in obj) {
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+    if (key === 'error') obj[key] = obj[key].message
+    else if (typeof obj[key] === 'object') scrub_errors(obj[key])
+  }
+}
+
+export async function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Promise<any> {
+  const expected = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`)
+  if (exclude_full_path) delete expected.full_path
+  return expected
+}
+
+export async function load_actual_evaluation (name: string): Promise<StoryEvaluation> {
+  const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`)
+  const display_path = `${name}.yaml`
+  const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml`
+  const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate()
+  scrub_errors(actual)
+  return actual
+}
diff --git a/tools/tests/tester/overall_result.test.ts b/tools/tests/tester/overall_result.test.ts
new file mode 100644
index 000000000..a269a6bd7
--- /dev/null
+++ b/tools/tests/tester/overall_result.test.ts
@@ -0,0 +1,13 @@
+import { overall_result } from '../../src/tester/helpers'
+import { type Evaluation, Result } from '../../src/tester/types/eval.types'
+
+function e (...results: Result[]): Evaluation[] {
+  return results.map(result => ({ result }))
+}
+test('overall_result', () => {
+  expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED, Result.ERROR))).toBe(Result.ERROR)
+  expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED))).toBe(Result.FAILED)
+  expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.SKIPPED)
+  expect(overall_result(e(Result.PASSED))).toBe(Result.PASSED)
+  expect(overall_result(e())).toBe(Result.PASSED)
+})