Add support for ML neural search. (#504)

Signed-off-by: dblock <[email protected]>

.cspell

@@ -59,6 +59,7 @@ gsub
 Gsub
 haasephonetik
 heteroscedastic
+hnsw
 homoscedastic
 hotthreads
 huggingface
@@ -74,6 +75,7 @@ kstem
 kuromoji
 Kuromoji
 languageset
+localstats
 Lovins
 lucene
 Lucene
@@ -176,6 +178,7 @@ tokenfilters
 translog
 Translog
 tubone
+Undeploys
 unigrams
 Unmanaged
 unmatch
@@ -186,5 +189,4 @@ urldecode
 vectory
 whoamiprotected
 wordnet
-Yrtsd
-localstats
+Yrtsd

CHANGELOG.md

@@ -69,7 +69,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471))
 - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476))
 - Added API spec for security plugin  ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271))
-- Added `/_plugins/_security/api/certificates/` to API spec  ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
+- Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
+- Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504))
 
 ### Changed
 

spec/namespaces/ml.yaml

@@ -53,6 +53,26 @@ paths:
       responses:
         '200':
           $ref: '#/components/responses/ml.delete_model@200'
+  /_plugins/_ml/models/{model_id}/_deploy:
+    post:
+      operationId: ml.deploy_model.0
+      x-operation-group: ml.deploy_model
+      description: Deploys a model.
+      parameters:
+        - $ref: '#/components/parameters/ml.deploy_model::path.model_id'
+      responses:
+        '200':
+          $ref: '#/components/responses/ml.deploy_model@200'
+  /_plugins/_ml/models/{model_id}/_undeploy:
+    post:
+      operationId: ml.undeploy_model.0
+      x-operation-group: ml.undeploy_model
+      description: Undeploys a model.
+      parameters:
+        - $ref: '#/components/parameters/ml.undeploy_model::path.model_id'
+      responses:
+        '200':
+          $ref: '#/components/responses/ml.undeploy_model@200'
   /_plugins/_ml/tasks/{task_id}:
     get:
       operationId: ml.get_task.0
@@ -161,6 +181,26 @@ components:
             required:
               - status
               - task_id
+    ml.deploy_model@200:
+      content:
+        application/json:
+          schema:
+            type: object
+            properties:
+              status:
+                type: string
+              task_id:
+                type: string
+              task_type:
+                type: string
+            required:
+              - status
+              - task_id
+    ml.undeploy_model@200:
+      content:
+        application/json:
+          schema:
+            $ref: '../schemas/ml._common.yaml#/components/schemas/UndeployModelResponse'
     ml.delete_model@200:
       content:
         application/json:
@@ -195,6 +235,18 @@ components:
       required: true
       schema:
         type: string
+    ml.deploy_model::path.model_id:
+      name: model_id
+      in: path
+      required: true
+      schema:
+        type: string
+    ml.undeploy_model::path.model_id:
+      name: model_id
+      in: path
+      required: true
+      schema:
+        type: string
     ml.get_task::path.task_id:
       name: task_id
       in: path

spec/schemas/_common.mapping.yaml

@@ -208,6 +208,7 @@ components:
         - $ref: '#/components/schemas/IntegerRangeProperty'
         - $ref: '#/components/schemas/IpRangeProperty'
         - $ref: '#/components/schemas/LongRangeProperty'
+        - $ref: '#/components/schemas/KnnVectorProperty'
     BinaryProperty:
       allOf:
         - $ref: '#/components/schemas/DocValuesPropertyBase'
@@ -1129,6 +1130,30 @@ components:
               type: boolean
             index:
               type: boolean
+    KnnVectorPropertyBase:
+      type: object
+      properties:
+        dimension:
+          type: number
+        method:
+          $ref: '#/components/schemas/KnnVectorMethod'
+      required:
+        - dimension
+    KnnVectorMethod:
+      type: object
+      properties:
+        name:
+          type: string
+        space_type:
+          type: string
+        engine:
+          type: string
+        parameters:
+          type: object
+          additionalProperties:
+            type: object
+      required:
+        - name
     DoubleRangeProperty:
       allOf:
         - $ref: '#/components/schemas/RangePropertyBase'
@@ -1184,6 +1209,17 @@ components:
                 - long_range
           required:
             - type
+    KnnVectorProperty:
+      allOf:
+        - $ref: '#/components/schemas/KnnVectorPropertyBase'
+        - type: object
+          properties:
+            type:
+              type: string
+              enum:
+                - knn_vector
+          required:
+            - type
     MatchType:
       type: string
       enum:

spec/schemas/_common.query_dsl.yaml

@@ -112,6 +112,8 @@ components:
           $ref: '#/components/schemas/MultiMatchQuery'
         nested:
           $ref: '#/components/schemas/NestedQuery'
+        neural:
+          $ref: '#/components/schemas/NeuralQuery'
         parent_id:
           $ref: '#/components/schemas/ParentIdQuery'
         percolate:
@@ -1222,6 +1224,32 @@ components:
           required:
             - path
             - query
+    NeuralQuery:
+      allOf:
+        - $ref: '#/components/schemas/QueryBase'
+        - type: object
+          additionalProperties:
+            $ref: '#/components/schemas/NeuralQueryVectorField'
+    NeuralQueryVectorField:
+      type: object
+      properties:
+        query_text:
+          type: string
+        query_image:
+          type: string
+          format: binary
+        model_id:
+          type: string
+        k:
+          type: integer
+        min_score:
+          type: number
+        max_distance:
+          type: number
+        filter:
+          $ref: '#/components/schemas/QueryContainer'
+      required:
+        - model_id
     ParentIdQuery:
       allOf:
         - $ref: '#/components/schemas/QueryBase'

spec/schemas/ml._common.yaml

@@ -139,5 +139,19 @@ components:
           format: int64
         is_async:
           type: boolean
+        error:
+          type: string
       required:
         - state
+    UndeployModelResponse:
+      type: object
+      additionalProperties:
+        $ref: '#/components/schemas/UndeployModelResponseModels'
+    UndeployModelResponseModels:
+      type: object
+      properties:
+        stats:
+          $ref: '#/components/schemas/UndeployModelResponseStats'
+    UndeployModelResponseStats:
+      type: object
+      additionalProperties: true