Skip to content

Commit

Permalink
Evaluate payload body in prologues.
Browse files Browse the repository at this point in the history
Signed-off-by: dblock <[email protected]>
  • Loading branch information
dblock committed Jan 7, 2025
1 parent 9c8431f commit 404b91f
Show file tree
Hide file tree
Showing 7 changed files with 167 additions and 50 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added `GET /_plugins/_ml/connectors/{connector_id}`, `_search`, `POST /_plugins/_ml/connectors/_search`, and `PUT /_plugins/_ml/connectors/{connector_id}` ([#764](https://github.com/opensearch-project/opensearch-api-specification/pull/764))
- Added the ability to skip an individual chapter test ([#765](https://github.com/opensearch-project/opensearch-api-specification/pull/765))
- Added uploading of test spec logs ([#767](https://github.com/opensearch-project/opensearch-api-specification/pull/767))
- Added support for evaluating response payloads in prologues and epilogues ([#772](https://github.com/opensearch-project/opensearch-api-specification/pull/772))

### Removed
- Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
Expand Down
22 changes: 3 additions & 19 deletions tools/src/tester/ChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,12 @@ import type OperationLocator from './OperationLocator'
import type SchemaValidator from './SchemaValidator'
import { type StoryOutputs } from './StoryOutputs'
import { ChapterOutput } from './ChapterOutput'
import { Operation, atomizeChangeset, diff } from 'json-diff-ts'
import _ from 'lodash'
import { Logger } from 'Logger'
import { sleep, to_json } from '../helpers'
import { APPLICATION_JSON } from "./MimeTypes";
import { APPLICATION_JSON } from "./MimeTypes"
import { ParsedChapter } from './types/parsed_story.types'
import ResponsePayloadEvaluator from './ResponsePayloadEvaluator'

export default class ChapterEvaluator {
private readonly logger: Logger
Expand Down Expand Up @@ -70,7 +70,7 @@ export default class ChapterEvaluator {
const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } }
const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED }
const payload_body_evaluation = status.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }

if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)

Expand Down Expand Up @@ -151,22 +151,6 @@ export default class ChapterEvaluator {
return result
}

#evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation {
if (expected_payload == null) return { result: Result.PASSED }
const payload = response.payload
this.logger.info(`${to_json(payload)}`)
const delta = atomizeChangeset(diff(expected_payload, payload))
const messages: string[] = _.compact(delta.map((value, _index, _array) => {
switch (value.type) {
case Operation.UPDATE:
return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
case Operation.REMOVE:
return `missing ${value.path.replace('$.', '')}='${value.value}'`
}
}))
return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
}

#evaluate_payload_schema(chapter: ParsedChapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
const content_type = chapter.response?.content_type ?? APPLICATION_JSON

Expand Down
39 changes: 39 additions & 0 deletions tools/src/tester/ResponsePayloadEvaluator.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import _ from "lodash"
import { Evaluation, Result } from './types/eval.types'
import { Logger } from "../Logger"
import { to_json } from "../helpers"
import { ActualResponse, Payload } from "./types/story.types"
import { atomizeChangeset, diff, Operation } from "json-diff-ts"

export default class ResponsePayloadEvaluator {
private readonly logger: Logger

constructor(logger: Logger) {
this.logger = logger
}

evaluate(response: ActualResponse, expected_payload?: Payload): Evaluation {
if (expected_payload == null) return { result: Result.PASSED }
const payload = response.payload
this.logger.info(`${to_json(payload)}`)
const delta = atomizeChangeset(diff(expected_payload, payload))
const messages: string[] = _.compact(delta.map((value, _index, _array) => {
switch (value.type) {
case Operation.UPDATE:
return `expected ${value.path.replace('$.', '')}='${value.oldValue}', got '${value.value}'`
case Operation.REMOVE:
return `missing ${value.path.replace('$.', '')}='${value.value}'`
}
}))
return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
}
}
30 changes: 16 additions & 14 deletions tools/src/tester/SupplementalChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,22 +7,23 @@
* compatible open source license.
*/

import _ from "lodash";
import { ChapterOutput } from "./ChapterOutput";
import ChapterReader from "./ChapterReader";
import { StoryOutputs } from "./StoryOutputs";
import { overall_result } from "./helpers";
import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types';
import { Logger } from "../Logger";
import { sleep, to_json } from "../helpers";
import { SupplementalChapter } from "./types/story.types";
import _ from "lodash"
import { ChapterOutput } from "./ChapterOutput"
import ChapterReader from "./ChapterReader"
import { StoryOutputs } from "./StoryOutputs"
import { overall_result } from "./helpers"
import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types'
import { Logger } from "../Logger"
import { sleep, to_json } from "../helpers"
import { Payload, SupplementalChapter } from "./types/story.types"
import ResponsePayloadEvaluator from './ResponsePayloadEvaluator'

export default class SupplementalChapterEvaluator {
private readonly _chapter_reader: ChapterReader;
private readonly logger: Logger;
private readonly _chapter_reader: ChapterReader
private readonly logger: Logger

constructor(chapter_reader: ChapterReader, logger: Logger) {
this._chapter_reader = chapter_reader;
this._chapter_reader = chapter_reader
this.logger = logger
}

Expand All @@ -49,10 +50,11 @@ export default class SupplementalChapterEvaluator {
const response = await this._chapter_reader.read(chapter, story_outputs)
const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)

const status = chapter.status ?? [200, 201]
const overall = status.includes(response.status) ? { result: Result.PASSED } : { result: Result.ERROR, message: response.message, error: response.error as Error }
const result: Result = overall_result(_.compact([overall, output_values_evaluation.evaluation]))
const response_payload: Payload | undefined = overall.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
const payload_body_evaluation = overall.result === Result.PASSED ? new ResponsePayloadEvaluator(this.logger).evaluate(response, response_payload) : { result: Result.SKIPPED }
const result: Result = overall_result(_.compact([overall, payload_body_evaluation, output_values_evaluation.evaluation]))

var evaluation_result: EvaluationWithOutput = { evaluation: { result } }
if (output_values_evaluation.output) { evaluation_result.output = output_values_evaluation.output }
Expand Down
38 changes: 21 additions & 17 deletions tools/src/tester/types/story.types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,6 @@ export type Version = string;
* via the `definition` "DistributionsList".
*/
export type DistributionsList = string[];
/**
* Number of times to retry on error.
*
*
* This interface was referenced by `Story`'s JSON-Schema
* via the `definition` "Retry".
*/
export type Retry = {
/**
* Number of retries.
*/
count: number;
/**
* Number of milliseconds to wait before retrying.
*/
wait?: number;
};
/**
* This interface was referenced by `Story`'s JSON-Schema
* via the `definition` "Chapter".
Expand All @@ -90,6 +73,9 @@ export type Chapter = ChapterRequest & {
* A brief description of the chapter.
*/
synopsis: string;
/**
* An explanation is provided to clarify why it has been skipped.
*/
pending?: string;
response?: ExpectedResponse;
warnings?: Warnings;
Expand Down Expand Up @@ -131,6 +117,7 @@ export interface ChapterRequest {
version?: Version;
distributions?: Distributions;
retry?: Retry;
response?: ExpectedResponse;
}
/**
* This interface was referenced by `Story`'s JSON-Schema
Expand Down Expand Up @@ -177,6 +164,23 @@ export interface Distributions {
included?: DistributionsList;
excluded?: DistributionsList;
}
/**
* Number of times to retry on error.
*
*
* This interface was referenced by `Story`'s JSON-Schema
* via the `definition` "Retry".
*/
export interface Retry {
/**
* Number of retries.
*/
count: number;
/**
* Number of milliseconds to wait before retrying.
*/
wait?: number;
}
/**
* This interface was referenced by `Story`'s JSON-Schema
* via the `definition` "ExpectedResponse".
Expand Down
39 changes: 39 additions & 0 deletions tools/tests/tester/ResponsePayloadEvaluator.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import { Result } from "tester/types/eval.types";
import ResponsePayloadEvaluator from "tester/ResponsePayloadEvaluator";
import { Logger } from "Logger";
import { ActualResponse } from "tester/types/story.types";

function create_response(payload: any): ActualResponse {
return {
status: 200,
content_type: 'application/json',
payload
}
}

describe('ResponsePayloadEvaluator', () => {
const evaluator = new ResponsePayloadEvaluator(new Logger())

describe('evaluate', () => {
test('succeeds without an expected payload', () => {
expect(evaluator.evaluate(create_response({}), undefined)).toEqual({ result: Result.PASSED })
})

test('fails with a non-matching payload', () => {
expect(evaluator.evaluate(create_response({}), { x: 1 })).toEqual({ result: Result.FAILED, message: "missing x='1'" })
})

test('succeeds with a matching payload', () => {
expect(evaluator.evaluate(create_response({ x: 1 }), { x: 1 })).toEqual({ result: Result.PASSED })
})
})
})
48 changes: 48 additions & 0 deletions tools/tests/tester/SupplementalChapterEvaluator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -86,5 +86,53 @@ describe('SupplementalChapterEvaluator', () => {
expect(result.overall.result).toEqual(Result.ERROR)
expect(count).toEqual(5)
})

test('a valid response payload', async () => {
mock.onAny().reply(200, '{"acknowledged":true}', { "content-type": "application/json" })

expect(
await supplemental_chapter_evaluator.evaluate({
path: '/test',
method: 'PUT',
request: {
payload: {}
},
response: {
status: 200,
payload: {
acknowledged: true
}
}
}, story_outputs)).toEqual({
title: 'PUT /test',
overall: {
result: Result.PASSED
}
})
})

test('an invalid response payload', async () => {
mock.onAny().reply(200, '{"acknowledged":false}', { "content-type": "application/json" })

expect(
await supplemental_chapter_evaluator.evaluate({
path: '/test',
method: 'PUT',
request: {
payload: {}
},
response: {
status: 200,
payload: {
acknowledged: true
}
}
}, story_outputs)).toEqual({
title: 'PUT /test',
overall: {
result: Result.FAILED
}
})
})
})
})

0 comments on commit 404b91f

Please sign in to comment.