Update model/search_pipeline/search_pipeline.smithy

Co-authored-by: Thomas Farr <[email protected]> Signed-off-by: Niyazbek Torekeldi <[email protected]>
Tokesh · Mar 16, 2024 · 6faa509 · 6faa509
1 parent d5c2961
commit 6faa509
Showing 1 changed file with 161 additions and 67 deletions.
diff --git a/model/search_pipeline/search_pipeline.smithy b/model/search_pipeline/search_pipeline.smithy
@@ -7,104 +7,198 @@
 $version: "2"
 namespace OpenSearch
 
-map SearchPipelineMap{
-    key: String,
+map SearchPipelineMap {
+    key: String
     value: SearchPipelineStructure
 }
 
-structure SearchPipelineStructure{
-    version: Integer,
-    request_processors: RequestProcessorsList,
-    response_processors: ResponseProcessorsList,
-    phase_results_processors: PhaseResultsProcessorsList
+structure SearchPipelineStructure {
+    version: Integer
+    request_processors: RequestProcessorList
+    response_processors: ResponseProcessorList
+    phase_results_processors: PhaseResultsProcessorList
 }
 
-list RequestProcessorsList {
-    member: RequestProcessors
+// === Common Properties ===
+
+@mixin
+structure ProcessorBase {
+    tag: String
+    description: String
 }
 
-list ResponseProcessorsList {
-    member: RequestProcessors
+@mixin
+structure IgnoreableProcessorBase with [ProcessorBase] {
+    ignore_failure: Boolean
 }
 
-list PhaseResultsProcessorsList {
-    member: PhaseResultsProcessors
+// === Request Processors ===
+
+list RequestProcessorList {
+    member: RequestProcessor
 }
 
-structure RequestProcessors {
-    filter_query: FilterQuery,
-    neural_query_enricher: NeuralQueryEnricher,
-    script: SearchScript
+union RequestProcessor {
+    filter_query: FilterQueryRequestProcessor
+    neural_query_enricher: NeuralQueryEnricherRequestProcessor
+    script: SearchScriptRequestProcessor
+    oversample: OversampleRequestProcessor
 }
 
-structure ResponseProcessors {
-    personalize_search_ranking: PersonalizeSearchRanking,
-    rename_field: RenameField,
+structure FilterQueryRequestProcessor with [IgnoreableProcessorBase] {
+    query: UserDefinedObjectStructure
 }
 
-structure PhaseResultsProcessors {
-    normalization: ScoreNormalization,
-    combination: ScoreCombination,
-    tag: String,
-    description: String,
-    ignore_failure: Boolean
+structure NeuralQueryEnricherRequestProcessor with [ProcessorBase] {
+    default_model_id: String
+    neural_field_default_id: NeuralFieldMap
 }
 
-structure ScoreNormalization{
-    technique: String
+map NeuralFieldMap {
+    key: String
+    value: String
 }
 
-structure ScoreCombination{
-    technique: String,
-    parameters: ScoreWeights
+structure SearchScriptRequestProcessor with [IgnoreableProcessorBase] {
+    @required
+    source: String
+
+    lang: String
 }
 
-list ScoreWeights{
-    member: Float
+structure OversampleRequestProcessor with [IgnoreableProcessorBase] {
+    @required
+    sample_factor: Float
+
+    content_prefix: String
 }
 
-structure FilterQuery {
-    query: UserDefinedObjectStructure,
-    tag: String,
-    description: String,
-    ignore_failure: Boolean
+// === Response Processors ===
+
+list ResponseProcessorList {
+    member: RequestProcessor
 }
 
-structure NeuralQueryEnricher {
-    default_model_id: String,
-    neural_field_default_id: NeuralFieldMap
-    tag: String,
-    description: String,
+union ResponseProcessor {
+    personalize_search_ranking: PersonalizeSearchRankingResponseProcessor
+    retrieval_augmented_generation: RetrievalAugmentedGenerationResponseProcessor
+    rename_field: RenameFieldResponseProcessor
+    rerank: RerankResponseProcessor
+    collapse: CollapseResponseProcessor
+    truncate_hits: TruncateHitsResponseProcessor
 }
 
-structure SearchScript {
-    source: String,
-    lang: String,
-    tag: String,
-    description: String,
-    ignore_failure: Boolean
+structure PersonalizeSearchRankingResponseProcessor with [IgnoreableProcessorBase] {
+    @required
+    campaign_arn: String
+
+    @required
+    recipe: String
+
+    @required
+    weight: Float
+
+    item_id_field: String
+
+    iam_role_arn: String
 }
 
-structure PersonalizeSearchRanking {
-    campaign_arn: String,
-    recipe: String,
-    weight: Float,
-    item_id_field: String,
-    iam_role_arn: String,
-    tag: String,
-    description: String,
-    ignore_failure: Boolean
+structure RetrievalAugmentedGenerationResponseProcessor with [ProcessorBase] {
+    @required
+    model_id: String
+
+    @required
+    context_field_list: ContextFieldList
+
+    system_prompt: String
+
+    user_instructions: String
 }
 
-structure RenameField {
-    field: String,
-    target_field: String,
-    tag: String,
-    description: String,
-    ignore_failure: Boolean
+list ContextFieldList {
+    member: String
 }
 
-map NeuralFieldMap {
-  key: String,
-  value: String
+structure RenameFieldResponseProcessor with [IgnoreableProcessorBase] {
+    @required
+    field: String
+
+    @required
+    target_field: String
+}
+
+structure RerankResponseProcessor with [IgnoreableProcessorBase] {
+    ml_opensearch: MLOpenSearchReranker
+    context: RerankContext
+}
+
+structure MLOpenSearchReranker {
+    @required
+    model_id: String
+}
+
+structure RerankContext {
+    @required
+    document_fields: DocumentFieldList
+}
+
+list DocumentFieldList {
+    member: String
+}
+
+structure CollapseResponseProcessor with [IgnoreableProcessorBase] {
+    @required
+    field: String
+
+    context_prefix: String
+}
+
+structure TruncateHitsResponseProcessor with [IgnoreableProcessorBase] {
+    target_size: Integer
+    context_prefix: String
+}
+
+// === Phase Results Processors ===
+
+list PhaseResultsProcessorList {
+    member: PhaseResultsProcessor
+}
+
+union PhaseResultsProcessor {
+    @jsonName("normalization-processor")
+    normalization_processor: NormalizationPhaseResultsProcessor
+}
+
+structure NormalizationPhaseResultsProcessor with [IgnoreableProcessorBase] {
+    normalization: ScoreNormalization
+    combination: ScoreCombination
 }
+
+structure ScoreNormalization {
+    technique: ScoreNormalizationTechnique
+}
+
+enum ScoreNormalizationTechnique {
+    MIN_MAX = "min_max"
+    L2 = "l2"
+}
+
+structure ScoreCombination {
+    technique: ScoreCombinationTechnique
+    parameters: ScoreWeights
+}
+
+enum ScoreCombinationTechnique {
+    ARITHMETIC_MEAN = "arithmetic_mean"
+    GEOMETRIC_MEAN = "geometric_mean"
+    HARMONIC_MEAN = "harmonic_mean"
+}
+
+structure ScoreCombinationParameters {
+    weights: ScoreWeights
+}
+
+list ScoreWeights {
+    member: Float
+}
+