From 8b6352fc1c7a7357082f0c61abbded0080e03447 Mon Sep 17 00:00:00 2001
From: dblock <dblock@amazon.com>
Date: Thu, 8 Aug 2024 18:53:29 -0400
Subject: [PATCH] Distinguish errors and errors.

Signed-off-by: dblock <dblock@amazon.com>
---
 tools/src/tester/ChapterEvaluator.ts                        | 2 +-
 tools/src/tester/ChapterReader.ts                           | 3 +--
 tools/tests/tester/fixtures/evals/error/chapter_error.yaml  | 1 -
 tools/tests/tester/fixtures/evals/error/prologue_error.yaml | 1 -
 4 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts
index 11ce2fe0c..ae73f90a3 100644
--- a/tools/src/tester/ChapterEvaluator.ts
+++ b/tools/src/tester/ChapterEvaluator.ts
@@ -122,7 +122,7 @@ export default class ChapterEvaluator {
 
   #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation {
     const expected_status = chapter.response?.status ?? 200
-    if (response.status === expected_status) return { result: Result.PASSED }
+    if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED }
 
     let result: Evaluation = {
       result: Result.ERROR,
diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts
index ec00cab3b..26dd02753 100644
--- a/tools/src/tester/ChapterReader.ts
+++ b/tools/src/tester/ChapterReader.ts
@@ -54,7 +54,7 @@ export default class ChapterReader {
       if (payload !== undefined) response.payload = payload
       this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(response.payload)}`)
     }).catch(e => {
-      if (e.response === undefined) {
+      if (e.response == null) {
         this.logger.info(`<= ERROR: ${e}`)
         response.message = e.message
         response.error = e
@@ -64,7 +64,6 @@ export default class ChapterReader {
         const payload = this.#deserialize_payload(e.response.data, response.content_type)
         if (payload !== undefined) response.payload = payload.error
         response.message = payload.error?.reason ?? e.response.statusText
-        response.error = e
         this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`)
       }
     })
diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml
index 450d9c5a3..fab1f0cc4 100644
--- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml
+++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml
@@ -33,7 +33,6 @@ chapters:
         result: ERROR
         message: 'Expected status 200, but received 404: application/json. no such index
           [undefined]'
-        error: Request failed with status code 404
       payload_body:
         result: SKIPPED
       payload_schema:
diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
index 5ff63bb87..d91beb82f 100644
--- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
+++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml
@@ -16,7 +16,6 @@ prologues:
     overall:
       result: ERROR
       message: no such index [does_not_exists]
-      error: Request failed with status code 404
 
 chapters:
   - title: This chapter be skipped.