From a54a5f5500ea901edcbaf610e9eac20968dc7080 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 9 Dec 2024 21:48:38 -0500 Subject: [PATCH] Added support for retry in prologues/epilogues, added replication tests. (#713) Signed-off-by: dblock Signed-off-by: Thomas Farr Co-authored-by: Thomas Farr --- CHANGELOG.md | 1 + tests/plugins/replication/autofollow.yaml | 18 +-- .../plugins/replication/autofollow_stats.yaml | 78 +++++++++++++ tests/plugins/replication/pause.yaml | 63 +++++++++++ tests/plugins/replication/resume.yaml | 69 ++++++++++++ tests/plugins/replication/start.yaml | 53 +++++++++ tests/plugins/replication/stats.yaml | 29 +---- tests/plugins/replication/status.yaml | 59 ++++++++++ .../replication/{index.yaml => update.yaml} | 44 +++----- tools/src/tester/ChapterReader.ts | 2 +- tools/src/tester/StoryEvaluator.ts | 4 +- .../tester/SupplementalChapterEvaluator.ts | 84 ++++++-------- tools/tests/tester/ChapterEvaluator.test.ts | 104 ++++++++++++++++++ .../SupplementalChapterEvaluator.test.ts | 90 +++++++++++++++ .../tests/tester/fixtures/specs/excerpt.yaml | 2 - tools/tests/tester/helpers.ts | 4 +- 16 files changed, 583 insertions(+), 121 deletions(-) create mode 100644 tests/plugins/replication/autofollow_stats.yaml create mode 100644 tests/plugins/replication/pause.yaml create mode 100644 tests/plugins/replication/resume.yaml create mode 100644 tests/plugins/replication/start.yaml create mode 100644 tests/plugins/replication/status.yaml rename tests/plugins/replication/{index.yaml => update.yaml} (78%) create mode 100644 tools/tests/tester/ChapterEvaluator.test.ts create mode 100644 tools/tests/tester/SupplementalChapterEvaluator.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index aecc07b15..3cbca08b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added missing `repository` query parameter to `/_cat/snapshots` ([#700](https://github.com/opensearch-project/opensearch-api-specification/pull/700)) - Added `sm` namespace API specifications ([#701](https://github.com/opensearch-project/opensearch-api-specification/pull/701)) - Added schema for `/_plugins/_knn/stats`, `/_plugins/_knn/models/{model_id}`, `_train` and `_search` ([#704](https://github.com/opensearch-project/opensearch-api-specification/pull/704)) +- Added `retry` support in `prologues` and `epilogues` ([#713](https://github.com/opensearch-project/opensearch-api-specification/pull/713)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/tests/plugins/replication/autofollow.yaml b/tests/plugins/replication/autofollow.yaml index 7f3cb8352..f13359011 100644 --- a/tests/plugins/replication/autofollow.yaml +++ b/tests/plugins/replication/autofollow.yaml @@ -1,6 +1,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml -description: Test replication APIs specs. +description: Test replication APIs. +version: '>= 1.1' prologues: - path: /{index} method: PUT @@ -10,6 +11,8 @@ prologues: payload: {} - path: /_bulk method: POST + parameters: + refresh: true request: content_type: application/x-ndjson payload: @@ -17,10 +20,6 @@ prologues: - {director: Bennett Miller, title: Moneyball, year: 2011} - {create: {_index: movies, _id: movie_1392215}} - {director: Badge Dale, title: The Secret Soldiers, year: 2016} - - path: /{index}/_refresh - method: POST - parameters: - index: movies - path: /_cluster/settings method: PUT request: @@ -31,7 +30,6 @@ prologues: leader-cluster: seeds: ['172.22.0.3:9300'] - path: /_plugins/_replication/{index}/_start - version: '>= 1.1' method: PUT parameters: index: movies-names @@ -58,7 +56,6 @@ epilogues: chapters: - synopsis: Create replication rule. path: /_plugins/_replication/_autofollow - version: '>= 1.1' method: POST request: payload: @@ -70,15 +67,8 @@ chapters: follower_cluster_role: admin response: status: 200 - - synopsis: Get auto-follow stats. - path: /_plugins/_replication/autofollow_stats - version: '>= 1.1' - method: GET - response: - status: 200 - synopsis: Delete replication rule. path: /_plugins/_replication/_autofollow - version: '>= 1.1' method: DELETE request: payload: diff --git a/tests/plugins/replication/autofollow_stats.yaml b/tests/plugins/replication/autofollow_stats.yaml new file mode 100644 index 000000000..c586bf9d9 --- /dev/null +++ b/tests/plugins/replication/autofollow_stats.yaml @@ -0,0 +1,78 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test replication APIs specs. +version: '>= 1.1' +prologues: + - path: /{index} + method: PUT + parameters: + index: movies + request: + payload: {} + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies, _id: movie_1392214}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies, _id: movie_1392215}} + - {director: Badge Dale, title: The Secret Soldiers, year: 2016} + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: movies-names + request: + payload: + leader_alias: leader-cluster + leader_index: movies + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access + - path: /_plugins/_replication/_autofollow + method: POST + request: + payload: + leader_alias: leader-cluster + name: movies + pattern: 'movies-*' + use_roles: + leader_cluster_role: leader_role + follower_cluster_role: admin +epilogues: + - path: /_plugins/_replication/_autofollow + method: DELETE + request: + payload: + leader_alias: leader-cluster + name: movies + status: [200, 404] + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: movies-names + request: + payload: {} + - path: /movies-names + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + method: GET + response: + status: 200 diff --git a/tests/plugins/replication/pause.yaml b/tests/plugins/replication/pause.yaml new file mode 100644 index 000000000..217efceae --- /dev/null +++ b/tests/plugins/replication/pause.yaml @@ -0,0 +1,63 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test pausing a replication. +version: '>= 1.1' +warnings: + multiple-paths-detected: false +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: music, _id: music_1392214}} + - {author: Selena Gomez, title: Love you like a love song, year: 2011} + - {create: {_index: music, _id: music_1392215}} + - {author: Justin Bieber, title: Baby, year: 2010} + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: music-names + request: + payload: + leader_alias: leader-cluster + leader_index: music + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: music-names + request: + payload: {} + - path: /music-names + method: DELETE + status: [200, 404] + - path: /music + method: DELETE + status: [200, 404] +chapters: + - synopsis: Pause replication. + path: /_plugins/_replication/{index}/_pause + method: POST + retry: + count: 3 + parameters: + index: music-names + request: + payload: {} + response: + status: 200 diff --git a/tests/plugins/replication/resume.yaml b/tests/plugins/replication/resume.yaml new file mode 100644 index 000000000..275e9fcd9 --- /dev/null +++ b/tests/plugins/replication/resume.yaml @@ -0,0 +1,69 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test pausing a replication. +version: '>= 1.1' +warnings: + multiple-paths-detected: false +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: music, _id: music_1392214}} + - {author: Selena Gomez, title: Love you like a love song, year: 2011} + - {create: {_index: music, _id: music_1392215}} + - {author: Justin Bieber, title: Baby, year: 2010} + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: music-names + request: + payload: + leader_alias: leader-cluster + leader_index: music + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access + - path: /_plugins/_replication/music-names/_pause + method: POST + retry: + count: 3 + request: + payload: {} +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: music-names + request: + payload: {} + - path: /music-names + method: DELETE + status: [200, 404] + - path: /music + method: DELETE + status: [200, 404] +chapters: + - synopsis: Resume replication. + path: /_plugins/_replication/{index}/_resume + method: POST + retry: + count: 3 + parameters: + index: music-names + request: + payload: {} + response: + status: 200 diff --git a/tests/plugins/replication/start.yaml b/tests/plugins/replication/start.yaml new file mode 100644 index 000000000..ccb4eb667 --- /dev/null +++ b/tests/plugins/replication/start.yaml @@ -0,0 +1,53 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test starting replication. +version: '>= 1.1' +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: Start replication. + path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: books-names + request: + payload: + leader_alias: leader-cluster + leader_index: books + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access + response: + status: 200 diff --git a/tests/plugins/replication/stats.yaml b/tests/plugins/replication/stats.yaml index c4e8102af..57f85baee 100644 --- a/tests/plugins/replication/stats.yaml +++ b/tests/plugins/replication/stats.yaml @@ -1,15 +1,14 @@ $schema: ../../../json_schemas/test_story.schema.yaml -description: Test replication APIs specs. +description: Test replication stats. +version: '>= 1.1' +warnings: + multiple-paths-detected: false prologues: - - path: /{index} - method: PUT - parameters: - index: music - request: - payload: {} - path: /_bulk method: POST + parameters: + refresh: true request: content_type: application/x-ndjson payload: @@ -17,10 +16,6 @@ prologues: - {author: Selena Gomez, title: Love you like a love song, year: 2011} - {create: {_index: music, _id: music_1392215}} - {author: Justin Bieber, title: Baby, year: 2010} - - path: /{index}/_refresh - method: POST - parameters: - index: music - path: /_cluster/settings method: PUT request: @@ -31,7 +26,6 @@ prologues: leader-cluster: seeds: ['172.22.0.3:9300'] - path: /_plugins/_replication/{index}/_start - version: '>= 1.1' method: PUT parameters: index: music-names @@ -56,29 +50,18 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get replication stats. - path: /_plugins/_replication/{index}/_status - version: '>= 1.1' - method: GET - parameters: - index: music-names - response: - status: 200 - synopsis: Get leader stats. path: /_plugins/_replication/leader_stats - version: '>= 1.1' method: GET response: status: 200 - synopsis: Get follower stats. path: /_plugins/_replication/follower_stats - version: '>= 1.1' method: GET response: status: 200 - synopsis: Get auto-follow stats. path: /_plugins/_replication/autofollow_stats - version: '>= 1.1' method: GET response: status: 200 \ No newline at end of file diff --git a/tests/plugins/replication/status.yaml b/tests/plugins/replication/status.yaml new file mode 100644 index 000000000..402883631 --- /dev/null +++ b/tests/plugins/replication/status.yaml @@ -0,0 +1,59 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test replication status. +version: '>= 1.1' +warnings: + multiple-paths-detected: false +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: music, _id: music_1392214}} + - {author: Selena Gomez, title: Love you like a love song, year: 2011} + - {create: {_index: music, _id: music_1392215}} + - {author: Justin Bieber, title: Baby, year: 2010} + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + cluster: + remote: + leader-cluster: + seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: music-names + request: + payload: + leader_alias: leader-cluster + leader_index: music + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: music-names + request: + payload: {} + - path: /music-names + method: DELETE + status: [200, 404] + - path: /music + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get replication status. + path: /_plugins/_replication/{index}/_status + method: GET + parameters: + index: music-names + response: + status: 200 diff --git a/tests/plugins/replication/index.yaml b/tests/plugins/replication/update.yaml similarity index 78% rename from tests/plugins/replication/index.yaml rename to tests/plugins/replication/update.yaml index a70a0b991..b3eae554c 100644 --- a/tests/plugins/replication/index.yaml +++ b/tests/plugins/replication/update.yaml @@ -1,15 +1,12 @@ $schema: ../../../json_schemas/test_story.schema.yaml -description: Test replication APIs specs. +description: Test updating settings for a started replication. +version: '>= 1.1' prologues: - - path: /{index} - method: PUT - parameters: - index: books - request: - payload: {} - path: /_bulk method: POST + parameters: + refresh: true request: content_type: application/x-ndjson payload: @@ -17,10 +14,6 @@ prologues: - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: books, _id: book_1392215}} - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} - - path: /{index}/_refresh - method: POST - parameters: - index: books - path: /_cluster/settings method: PUT request: @@ -30,6 +23,17 @@ prologues: remote: leader-cluster: seeds: ['172.22.0.3:9300'] + - path: /_plugins/_replication/{index}/_start + method: PUT + parameters: + index: books-names + request: + payload: + leader_alias: leader-cluster + leader_index: books + use_roles: + leader_cluster_role: all_access + follower_cluster_role: all_access epilogues: - path: /_plugins/_replication/{index}/_stop method: POST @@ -44,24 +48,8 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Start replication. - path: /_plugins/_replication/{index}/_start - version: '>= 1.1' - method: PUT - parameters: - index: books-names - request: - payload: - leader_alias: leader-cluster - leader_index: books - use_roles: - leader_cluster_role: all_access - follower_cluster_role: all_access - response: - status: 200 - - synopsis: Update settings. + - synopsis: Update replication settings. path: /_plugins/_replication/{index}/_update - version: '>= 1.1' method: PUT parameters: index: books-names diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index ad34d715b..fd78f7cc3 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -62,7 +62,7 @@ export default class ChapterReader { response.status = e.response.status response.content_type = e.response.headers['content-type']?.split(';')[0] response.payload = this.#deserialize_payload(e.response.data, response.content_type) - response.message = response.payload.error?.reason ?? e.response.statusText + response.message = response.payload?.error?.reason ?? e.response.statusText this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } }) diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index e464f3f60..ce02e53ae 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -140,8 +140,8 @@ export default class StoryEvaluator { if (dry_run) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) } else { - const { evaluation, evaluation_error } = await this._supplemental_chapter_evaluator.evaluate(chapter, story_outputs) - has_errors = has_errors || evaluation_error + const evaluation = await this._supplemental_chapter_evaluator.evaluate(chapter, story_outputs) + has_errors = has_errors || (evaluation.overall.result !== Result.PASSED && evaluation.overall.result !== Result.SKIPPED) if (evaluation.output !== undefined && chapter.id !== undefined) { story_outputs.set_chapter_output(chapter.id, evaluation.output) } diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index faa6c1004..cecf93407 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -12,10 +12,10 @@ import { ChapterOutput } from "./ChapterOutput"; import ChapterReader from "./ChapterReader"; import { StoryOutputs } from "./StoryOutputs"; import { overall_result } from "./helpers"; -import { ChapterEvaluation, Evaluation, Result } from "./types/eval.types"; +import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types'; import { SupplementalChapter } from "./types/story.types"; import { Logger } from "../Logger"; -import { to_json } from "../helpers"; +import { sleep, to_json } from "../helpers"; export default class SupplementalChapterEvaluator { private readonly _chapter_reader: ChapterReader; @@ -26,63 +26,47 @@ export default class SupplementalChapterEvaluator { this.logger = logger } - async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<{ evaluation: ChapterEvaluation, evaluation_error: boolean }> { + async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise { const title = `${chapter.method} ${chapter.path}` + + let tries = chapter.retry && chapter.retry?.count > 0 ? chapter.retry.count + 1 : 1 + let chapter_evaluation: EvaluationWithOutput + + do { + chapter_evaluation = await this.#evaluate(chapter, story_outputs) + if (chapter_evaluation.evaluation.result === Result.PASSED) break + if (--tries == 0) break + this.logger.info(`Failed, retrying, ${tries == 1 ? '1 retry left' : `${tries} retries left`} ...`) + await sleep(chapter.retry?.wait ?? 1000) + } while (tries > 0) + + var result: ChapterEvaluation = { title, overall: chapter_evaluation.evaluation } + if (chapter_evaluation.output) result.output = chapter_evaluation.output + return result + } + + async #evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise { const response = await this._chapter_reader.read(chapter, story_outputs) - const status = chapter.status ?? [200, 201] const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output) if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) - let response_evaluation: ChapterEvaluation - const passed_evaluation = { title, overall: { result: Result.PASSED } } - if (status.includes(response.status)) { - response_evaluation = passed_evaluation - } else { - response_evaluation = { - title, - overall: { - result: Result.ERROR, - message: response.message, - error: response.error as Error - } - } - } - if (output_values_evaluation.output) { - response_evaluation.output = output_values_evaluation.output - } + const status = chapter.status ?? [200, 201] + const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error } + const result: Result = overall_result(_.compact([overall, output_values_evaluation.evaluation])) - const result = overall_result(_.compact([ - response_evaluation.overall, - output_values_evaluation.evaluation - ])) + var evaluation_result: EvaluationWithOutput = { evaluation: { result } } + if (output_values_evaluation.output) { evaluation_result.output = output_values_evaluation.output } - if (result === Result.PASSED) { - return { evaluation: passed_evaluation, evaluation_error: false } - } else { + if (result !== Result.PASSED) { const message_segments = [] - - if (response_evaluation.overall.result === Result.ERROR) { - message_segments.push(`${response_evaluation.overall.message}`) - } - - if (output_values_evaluation.evaluation.message !== undefined && output_values_evaluation.evaluation.result === Result.ERROR) { + if (response.message !== undefined) { message_segments.push(`${response.message}`) } + if (output_values_evaluation.evaluation.result === Result.ERROR) { message_segments.push(`${output_values_evaluation.evaluation.message}`) } - - const message = message_segments.join('\n') - - var overall: Evaluation = { result: Result.ERROR, message } - if (response.error !== undefined) overall.error = response.error as Error - - var evaluation: ChapterEvaluation = { - title, overall - } - - if (output_values_evaluation.output) { - evaluation.output = output_values_evaluation.output - } - - return { evaluation, evaluation_error: true } + if (message_segments.length > 0) evaluation_result.evaluation.message = message_segments.join('\n') + if (response.error) { evaluation_result.evaluation.error = response.error as Error } } + + return evaluation_result } -} +} \ No newline at end of file diff --git a/tools/tests/tester/ChapterEvaluator.test.ts b/tools/tests/tester/ChapterEvaluator.test.ts new file mode 100644 index 000000000..dc4da95a6 --- /dev/null +++ b/tools/tests/tester/ChapterEvaluator.test.ts @@ -0,0 +1,104 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import axios from "axios"; +import { StoryOutputs } from 'tester/StoryOutputs'; +import { construct_tester_components } from './helpers' +import AxiosMockAdapter from "axios-mock-adapter"; +import { Result } from "tester/types/eval.types"; + +describe('ChapterEvaluator', () => { + var mock = new AxiosMockAdapter(axios) + const { chapter_evaluator } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') + + afterEach(() => { + mock.reset() + }) + + test('instance', () => { + expect(chapter_evaluator).toBeDefined(); + }); + + describe('evaluate', () => { + const story_outputs = new StoryOutputs() + + test('a path not found in spec', async () => { + expect( + await chapter_evaluator.evaluate({ + synopsis: 'Perform a GET on an invalid path.', + path: '/invalid', + method: 'GET' + }, false, story_outputs)).toEqual( + { + title: 'Perform a GET on an invalid path.', + overall: { + result: Result.FAILED, + message: 'Operation "GET /invalid" not found in the spec.' + } + } + ) + }) + + test('a successful response', async () => { + mock.onAny().reply(200, '{"acknowledged":true}', { "content-type": "application/json" }) + + expect( + await chapter_evaluator.evaluate({ + synopsis: 'Perform a PUT /{index}.', + path: '/{index}', + method: 'PUT', + parameters: { + index: 'test' + }, + request: { + payload: {} + } + }, false, story_outputs)).toEqual( + { + title: 'Perform a PUT /{index}.', + path: 'PUT /{index}', + operation: { method: 'PUT', path: '/{index}' }, + request: { parameters: { index: { result: 'PASSED' } }, request: { result: 'PASSED' } }, + response: { output_values: { result: 'SKIPPED' }, payload_body: { result: 'PASSED' }, payload_schema: { result: 'PASSED' }, status: { result: 'PASSED' } }, + overall: { + result: Result.PASSED + } + } + ) + }) + + test('retries', async () => { + var count = 0 + + mock.onAny().reply((_config) => { + count += 1 + return [400, 'Bad Request'] + }) + + var result = await chapter_evaluator.evaluate({ + synopsis: 'Perform a PUT /{index}.', + path: '/{index}', + method: 'PUT', + retry: { + count: 4, + wait: 0 + }, + parameters: { + index: 'test' + }, + request: { + payload: {} + } + }, false, story_outputs) + + expect(result.overall.result).toEqual(Result.ERROR) + expect(count).toEqual(5) + }) + }) +}) diff --git a/tools/tests/tester/SupplementalChapterEvaluator.test.ts b/tools/tests/tester/SupplementalChapterEvaluator.test.ts new file mode 100644 index 000000000..56bb01eab --- /dev/null +++ b/tools/tests/tester/SupplementalChapterEvaluator.test.ts @@ -0,0 +1,90 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import axios from "axios"; +import { StoryOutputs } from 'tester/StoryOutputs'; +import { construct_tester_components } from './helpers' +import AxiosMockAdapter from "axios-mock-adapter"; +import { Result } from "tester/types/eval.types"; + +describe('SupplementalChapterEvaluator', () => { + var mock = new AxiosMockAdapter(axios) + const { supplemental_chapter_evaluator } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') + + afterEach(() => { + mock.reset() + }) + + test('instance', () => { + expect(supplemental_chapter_evaluator).toBeDefined(); + }); + + describe('evaluate', () => { + const story_outputs = new StoryOutputs() + + test('a path not found in spec', async () => { + expect( + await supplemental_chapter_evaluator.evaluate({ + path: '/invalid', + method: 'GET' + }, story_outputs)).toEqual( + { + title: 'GET /invalid', + overall: { + result: Result.ERROR, + } + } + ) + }) + + test('a successful response', async () => { + mock.onAny().reply(200, '{"acknowledged":true}', { "content-type": "application/json" }) + + expect( + await supplemental_chapter_evaluator.evaluate({ + path: '/test', + method: 'PUT', + request: { + payload: {} + } + }, story_outputs)).toEqual( + { + title: 'PUT /test', + overall: { + result: Result.PASSED + } + } + ) + }) + + test('retries', async () => { + var count = 0 + + mock.onAny().reply((_config) => { + count += 1 + return [400, 'Bad Request'] + }) + + var result = await supplemental_chapter_evaluator.evaluate({ + path: '/test', + method: 'PUT', + retry: { + count: 4, + wait: 0 + }, + request: { + payload: {} + } + }, story_outputs) + + expect(result.overall.result).toEqual(Result.ERROR) + expect(count).toEqual(5) + }) + }) +}) diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml index 6b28883ab..fcc19a666 100644 --- a/tools/tests/tester/fixtures/specs/excerpt.yaml +++ b/tools/tests/tester/fixtures/specs/excerpt.yaml @@ -179,8 +179,6 @@ components: type: boolean required: - acknowledged - - index - - shards_acknowledged parameters: cat.health::query.format: in: query diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 399c1359e..a8705bd2f 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -30,7 +30,8 @@ export function construct_tester_components (spec_path: string): { opensearch_http_client: OpenSearchHttpClient chapter_reader: ChapterReader schema_validator: SchemaValidator - chapter_evaluator: ChapterEvaluator + chapter_evaluator: ChapterEvaluator, + supplemental_chapter_evaluator: SupplementalChapterEvaluator, story_validator: StoryValidator story_evaluator: StoryEvaluator result_logger: ResultLogger @@ -62,6 +63,7 @@ export function construct_tester_components (spec_path: string): { chapter_reader, schema_validator, chapter_evaluator, + supplemental_chapter_evaluator, story_validator, story_evaluator, result_logger,