diff --git a/.cspell b/.cspell index 65a801cbf..3656f245c 100644 --- a/.cspell +++ b/.cspell @@ -91,7 +91,6 @@ mmapfs mmdb mokotoff Moneyball -Moneyball msearch msmarco mtermvectors @@ -138,6 +137,9 @@ Reindex relo reloadcerts remotestore +rerank +Rerank +Reranker rethrottle Rethrottle rolesmapping diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f7db7db..f99a07a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) +- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) ### Changed diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index 94fd37470..ebb14ee9a 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -23,7 +23,7 @@ components: response_processors: type: array items: - $ref: '#/components/schemas/RequestProcessor' + $ref: '#/components/schemas/ResponseProcessor' phase_results_processors: type: array items: @@ -187,6 +187,219 @@ components: type: string required: - sample_factor + ResponseProcessor: + oneOf: + - type: object + title: personalize_search_ranking + properties: + personalize_search_ranking: + $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor' + required: + - personalize_search_ranking + - type: object + title: retrieval_augmented_generation + properties: + retrieval_augmented_generation: + $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor' + required: + - retrieval_augmented_generation + - type: object + title: rename_field + properties: + rename_field: + $ref: '#/components/schemas/RenameFieldResponseProcessor' + required: + - rename_field + - type: object + title: rerank + properties: + rerank: + $ref: '#/components/schemas/RerankResponseProcessor' + required: + - rerank + - type: object + title: collapse + properties: + collapse: + $ref: '#/components/schemas/CollapseResponseProcessor' + required: + - collapse + - type: object + title: truncate_hits + properties: + truncate_hits: + $ref: '#/components/schemas/TruncateHitsResponseProcessor' + required: + - truncate_hits + - type: object + title: sort + properties: + sort: + $ref: '#/components/schemas/SortResponseProcessor' + required: + - sort + - type: object + title: split + properties: + split: + $ref: '#/components/schemas/SplitResponseProcessor' + required: + - split + PersonalizeSearchRankingResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + campaign_arn: + type: string + recipe: + type: string + weight: + type: number + format: float + item_id_field: + type: string + iam_role_arn: + type: string + required: + - campaign_arn + - recipe + - weight + RetrievalAugmentedGenerationResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + model_id: + type: string + context_field_list: + type: array + items: + type: string + system_prompt: + type: string + user_instructions: + type: string + required: + - context_field_list + - model_id + RenameFieldResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + target_field: + type: string + required: + - field + - target_field + RerankContext: + type: object + properties: + document_fields: + type: array + items: + type: string + required: + - document_fields + MLOpenSearchReranker: + type: object + properties: + model_id: + type: string + required: + - model_id + RerankResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + ml_opensearch: + $ref: '#/components/schemas/MLOpenSearchReranker' + context: + $ref: '#/components/schemas/RerankContext' + CollapseResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + context_prefix: + type: string + required: + - field + TruncateHitsResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + target_size: + type: integer + format: int32 + context_prefix: + type: string + SortResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + order: + type: string + target_field: + type: string + required: + - field + SplitResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + separator: + type: string + preserve_trailing: + type: boolean + target_field: + type: string + required: + - field + - separator PhaseResultsProcessor: oneOf: - type: object diff --git a/tests/default/_core/search/pipeline/request_processor/filter_query.yaml b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml new file mode 100644 index 000000000..e2d2cfb79 --- /dev/null +++ b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /_bulk + method: POST + parameters: + refresh: 'true' + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +epilogues: + - path: /_search/pipeline/filter_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: filter_pipeline + request: + payload: + request_processors: + - filter_query: + tag: tag + description: This processor restricts searches to 20th century movies. + query: + range: + year: + lte: 2000 + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: filter_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: filter_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive + director: Nicolas Winding Refn + year: 1960 diff --git a/tests/default/_core/search/pipeline/response_processor/rename_field.yaml b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml new file mode 100644 index 000000000..388d1414a --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml @@ -0,0 +1,62 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + name: Drive + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - rename_field: + field: name + target_field: title + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive diff --git a/tests/default/_core/search/pipeline/response_processor/sort.yaml b/tests/default/_core/search/pipeline/response_processor/sort.yaml new file mode 100644 index 000000000..837501f66 --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/sort.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.16' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - sort: + field: names + order: asc + target_field: sorted_names + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + sorted_names: + - '1984' + - Drive + - Moneyball \ No newline at end of file diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 0d1d89a22..2d8b41ec1 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -155,7 +155,7 @@ chapters: - {create: {_index: movies}} - {director: Bennett Miller, title: Moneyball, year: 2011} - {create: {_index: movies}} - - {author: Nicolas Winding Refn, title: Drive, year: 1960} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} response: status: 200 - synopsis: Search.