Skip to content

Commit

Permalink
Added schema for /_plugins/_knn/stats, `/_plugins/_knn/models/{mode…
Browse files Browse the repository at this point in the history
…l_id}`, `_train` and `_search`. (#708)

Signed-off-by: dblock <[email protected]>
Signed-off-by: Thomas Farr <[email protected]>
Co-authored-by: Thomas Farr <[email protected]>
  • Loading branch information
dblock and Xtansia authored Dec 10, 2024
1 parent 807fa8f commit 8b9a491
Show file tree
Hide file tree
Showing 7 changed files with 378 additions and 42 deletions.
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added request body to `_search_shards` API specs ([#709](https://github.com/opensearch-project/opensearch-api-specification/pull/709))
- Added NodeAttributes as additional property in `shard_stores` API specs ([#715](https://github.com/opensearch-project/opensearch-api-specification/pull/715))
- Added missing `repository` query parameter to `/_cat/snapshots` ([#700](https://github.com/opensearch-project/opensearch-api-specification/pull/700))
- Added `sm` namespace API specifications ([#701](https://github.com/opensearch-project/opensearch-api-specification/pull/701))
- Added schema for `/_plugins/_knn/stats`, `/_plugins/_knn/models/{model_id}`, `_train` and `_search` ([#704](https://github.com/opensearch-project/opensearch-api-specification/pull/704))

### Removed
- Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
Expand Down Expand Up @@ -167,7 +169,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added API spec for `adjust_pure_negative` for bool queries ([#641](https://github.com/opensearch-project/opensearch-api-specification/pull/641))
- Added a spec style checker [#620](https://github.com/opensearch-project/opensearch-api-specification/pull/620).
- Added `remote_store` to node `Stats` ([#643](https://github.com/opensearch-project/opensearch-api-specification/pull/643))
- Added `sm` namespace API specifications ([#701](https://github.com/opensearch-project/opensearch-api-specification/pull/701))

### Changed

Expand Down
56 changes: 23 additions & 33 deletions spec/namespaces/knn.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -251,45 +251,28 @@ components:
content:
application/json:
schema:
type: object
properties:
training_index:
type: string
training_field:
type: string
dimension:
type: integer
format: int32
max_training_vector_count:
type: integer
format: int32
search_size:
type: integer
format: int32
description:
type: string
mode:
type: string
compression_level:
type: string
method:
type: string
spaceType:
type: string
required:
- dimension
- training_field
- training_index
required: true
$ref: '../schemas/knn._common.yaml#/components/schemas/TrainedModel'
responses:
knn.delete_model@200: {}
knn.delete_model@200:
content:
application/json:
schema:
$ref: '../schemas/knn._common.yaml#/components/schemas/DeletedModel'
knn.get_model@200:
content:
application/json:
schema:
type: object
knn.search_models@200: {}
knn.stats@200: {}
knn.search_models@200:
content:
application/json:
schema:
$ref: '../schemas/_core.search.yaml#/components/schemas/ResponseBody'
knn.stats@200:
content:
application/json:
schema:
$ref: '../schemas/knn._common.yaml#/components/schemas/Stats'
knn.train_model@200:
content:
application/json:
Expand Down Expand Up @@ -681,12 +664,19 @@ components:
- graph_memory_usage_percentage
- graph_query_errors
- graph_query_requests
- graph_stats
- hit_count
- indexing_from_model_degraded
- indices_in_cache
- knn_query_requests
- knn_query_with_filter_requests
- load_exception_count
- load_success_count
- lucene_initialized
- max_distance_query_requests
- max_distance_query_with_filter_requests
- min_score_query_requests
- min_score_query_with_filter_requests
- miss_count
- model_index_status
- nmslib_initialized
Expand Down
161 changes: 161 additions & 0 deletions spec/schemas/knn._common.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,164 @@ components:
- always
- missing
- popular
Stats:
type: object
properties:
_nodes:
$ref: '_common.yaml#/components/schemas/NodeStatistics'
cluster_name:
$ref: '_common.yaml#/components/schemas/Name'
circuit_breaker_triggered:
type: boolean
model_index_status:
oneOf:
- $ref: '_common.yaml#/components/schemas/HealthStatusCapitalized'
- type: 'null'
nodes:
type: object
additionalProperties:
$ref: '#/components/schemas/NodeStats'
NodeStats:
type: object
properties:
max_distance_query_with_filter_requests:
type: number
graph_memory_usage_percentage:
$ref: '_common.yaml#/components/schemas/PercentageNumber'
graph_query_requests:
type: number
graph_memory_usage:
type: number
cache_capacity_reached:
type: boolean
load_success_count:
type: number
training_memory_usage:
type: number
indices_in_cache:
type: object
additionalProperties: true
script_query_errors:
type: number
hit_count:
type: number
knn_query_requests:
type: number
total_load_time:
type: number
miss_count:
type: number
min_score_query_requests:
type: number
knn_query_with_filter_requests:
type: number
training_memory_usage_percentage:
$ref: '_common.yaml#/components/schemas/PercentageNumber'
max_distance_query_requests:
type: number
lucene_initialized:
type: boolean
graph_index_requests:
type: number
faiss_initialized:
type: boolean
load_exception_count:
type: number
training_errors:
type: number
min_score_query_with_filter_requests:
type: number
eviction_count:
type: number
nmslib_initialized:
type: boolean
script_compilations:
type: number
script_query_requests:
type: number
graph_stats:
$ref: '#/components/schemas/GraphStats'
graph_query_errors:
type: number
indexing_from_model_degraded:
type: boolean
graph_index_errors:
type: number
training_requests:
type: number
script_compilation_errors:
type: number
GraphStats:
type: object
properties:
merge:
$ref: '#/components/schemas/GraphMergeStats'
refresh:
$ref: '#/components/schemas/GraphRefreshStats'
GraphRefreshStats:
type: object
properties:
total:
type: number
total_time_in_millis:
$ref: '_common.yaml#/components/schemas/DurationValueUnitMillis'
GraphMergeStats:
type: object
properties:
current:
type: number
total:
type: number
total_time_in_millis:
$ref: '_common.yaml#/components/schemas/DurationValueUnitMillis'
current_docs:
type: number
total_docs:
type: number
total_size_in_bytes:
$ref: '_common.yaml#/components/schemas/ByteCount'
current_size_in_bytes:
$ref: '_common.yaml#/components/schemas/ByteCount'
TrainedModel:
type: object
properties:
training_index:
type: string
training_field:
type: string
dimension:
type: integer
format: int32
max_training_vector_count:
type: integer
format: int32
search_size:
type: integer
format: int32
description:
type: string
mode:
type: string
compression_level:
type: string
method:
type: string
spaceType:
type: string
required:
- dimension
- training_field
- training_index
DeletedModel:
type: object
properties:
model_id:
type: string
result:
type: string
enum:
- deleted
- error
required:
- model_id
- result
61 changes: 61 additions & 0 deletions tests/default/knn/models.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
$schema: ../../../json_schemas/test_story.schema.yaml

description: Delete a k-NN model.
version: '>= 2.17'
prologues:
- method: PUT
path: /movies
request:
payload:
settings:
index:
knn: true
mappings:
properties:
recommendation_vector:
type: knn_vector
dimension: 8
status: [200]
- method: POST
path: /_bulk
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {index: {_index: movies, _id: '1'}}
- {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2}
- {index: {_index: movies, _id: '2'}}
- {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1}
status: [200]
- method: POST
path: /_plugins/_knn/models/model-1/_train
request:
payload:
training_index: movies
training_field: recommendation_vector
dimension: 8
max_training_vector_count: 1200
search_size: 100
description: Test model
mode: on_disk
compression_level: 32x
spaceType: l2
status: [200]
epilogues:
- path: /movies
method: DELETE
status: [200, 404]
- path: /_plugins/_knn/models/model-1
method: DELETE
status: [200, 404]
chapters:
- synopsis: Delete a trained model.
method: DELETE
path: /_plugins/_knn/models/{model_id}
parameters:
model_id: model-1
retry:
count: 3
response:
status: 200
70 changes: 70 additions & 0 deletions tests/default/knn/models/search.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
$schema: ../../../../json_schemas/test_story.schema.yaml

description: Search for a k-NN model.
version: '>= 2.17'
prologues:
- method: PUT
path: /movies
request:
payload:
settings:
index:
knn: true
mappings:
properties:
recommendation_vector:
type: knn_vector
dimension: 8
status: [200]
- method: POST
path: /_bulk
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {index: {_index: movies, _id: '1'}}
- {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2}
- {index: {_index: movies, _id: '2'}}
- {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1}
status: [200]
- method: POST
path: /_plugins/_knn/models/model-1/_train
request:
payload:
training_index: movies
training_field: recommendation_vector
dimension: 8
max_training_vector_count: 1200
search_size: 100
description: Test model
mode: on_disk
compression_level: 32x
spaceType: l2
status: [200]
epilogues:
- path: /movies
method: DELETE
status: [200, 404]
- path: /_plugins/_knn/models/model-1
method: DELETE
status: [200, 404]
chapters:
- synopsis: Search models (GET).
method: GET
path: /_plugins/_knn/models/_search
request:
payload:
query:
match_all: {}
response:
status: 200
- synopsis: Search models (POST).
method: POST
path: /_plugins/_knn/models/_search
request:
payload:
query:
match_all: {}
response:
status: 200
Loading

0 comments on commit 8b9a491

Please sign in to comment.