From edef4e03ba65b715d56e3ca3f15750ae151de98f Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Sat, 7 Sep 2024 03:49:45 +1200 Subject: [PATCH 01/73] Fix some integer/long typings (#557) --- spec/schemas/_common.yaml | 29 ++++++++++++++++------------- spec/schemas/ml._common.yaml | 2 +- spec/schemas/tasks._common.yaml | 3 ++- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index e2528326e..7e55ff907 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -138,7 +138,7 @@ components: - $ref: '#/components/schemas/UnitMillis' UnitMillis: description: Time unit for milliseconds. - type: number + type: integer format: int64 DurationLarge: description: |- @@ -173,10 +173,10 @@ components: description: Path to field or array of paths. Some API's support wildcards in the path to select multiple fields. type: string SequenceNumber: - type: number + type: integer format: int64 VersionNumber: - type: number + type: integer format: int64 SortResults: type: array @@ -276,7 +276,7 @@ components: - successful - total uint: - type: number + type: integer ShardFailure: type: object properties: @@ -287,7 +287,7 @@ components: reason: $ref: '#/components/schemas/ErrorCause' shard: - type: number + type: integer status: type: string required: @@ -325,7 +325,8 @@ components: - $ref: '#/components/schemas/UnitNanos' UnitNanos: description: Time unit for nanoseconds. - type: number + type: integer + format: int64 ScrollId: type: string Routing: @@ -1318,13 +1319,13 @@ components: $ref: '#/components/schemas/ErrorCause' total: description: Total number of nodes selected by the request. - type: number + type: integer successful: description: Number of nodes that responded successfully to the request. - type: number + type: integer failed: description: Number of nodes that rejected the request or failed to respond. If this value is not 0, a reason for the rejection or failure is included in the response. - type: number + type: integer required: - failed - successful @@ -1339,7 +1340,7 @@ components: _index: $ref: '#/components/schemas/IndexName' _primary_term: - type: number + type: integer format: int64 result: $ref: '#/components/schemas/Result' @@ -1404,9 +1405,11 @@ components: type: object properties: bulk: - type: number + type: integer + format: int64 search: - type: number + type: integer + format: int64 required: - bulk - search @@ -1418,7 +1421,7 @@ components: type: object properties: task_id: - type: number + type: integer node_id: $ref: '#/components/schemas/NodeId' status: diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 219002dbe..e95621511 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -84,7 +84,7 @@ components: type: string description: The model group name. latest_version: - type: number + type: integer description: The latest version. description: type: string diff --git a/spec/schemas/tasks._common.yaml b/spec/schemas/tasks._common.yaml index b6d85e989..8e5399cb4 100644 --- a/spec/schemas/tasks._common.yaml +++ b/spec/schemas/tasks._common.yaml @@ -65,7 +65,8 @@ components: additionalProperties: type: string id: - type: number + type: integer + format: int64 node: $ref: '_common.yaml#/components/schemas/NodeId' running_time: From 9193d58ec90a5328e866f7eb67e019d1176d96ec Mon Sep 17 00:00:00 2001 From: Junwei Dai <59641585+junweid62@users.noreply.github.com> Date: Fri, 6 Sep 2024 10:54:57 -0700 Subject: [PATCH 02/73] Add flow_framework namespace specs (#553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add flow framework create Signed-off-by: Junwei Dai * add flow framework delete Signed-off-by: Junwei Dai * add flow framework put and get Signed-off-by: Junwei Dai * add changelog info Signed-off-by: Junwei Dai * fix mistype error refactor: 1.add epilogues that deletes the workflow if it was created. 2.rewrite all the 'whether' 3. remove empty lines 4. add validation enum Signed-off-by: Junwei Dai * refactor: add version to flow framework and test Signed-off-by: Junwei Dai * refactor: add delete path param, add delete test, would also delete the workflow previously created Signed-off-by: Junwei Dai * refactor: Add schema for 4xx response Signed-off-by: Junwei Dai * refactor: Add eslint ignore 'reprovision' Signed-off-by: Junwei Dai * refactor: remove empty line Signed-off-by: Junwei Dai * refactor: fix merge conflict error Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai * refactor: fix merge conflict error Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai * Update tests/default/flow_framework/workflow.yaml Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> * refactor: 1.change flow_framework._common to flow_framework._errors 2.change exception to Error for consistency Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai * add use_case parameter, add reprovision to .cspell Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai * refactor: fix eslint Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai * refactor: remove comment Signed-off-by: Junwei Dai * refactor: fix lint spec error Signed-off-by: Junwei Dai * refactor: fix lint spec error Signed-off-by: Junwei Dai * refactor: fix lint spec error Signed-off-by: Junwei Dai * refactor: fix lint spec error， add flow_framework.common file Signed-off-by: Junwei Dai --------- Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> Co-authored-by: Junwei Dai Co-authored-by: Daniel (dB.) Doubrovkine --- .cspell | 3 +- CHANGELOG.md | 1 + spec/namespaces/flow_framework.yaml | 225 +++++++++++++++++++++ spec/schemas/flow_framework.common.yaml | 127 ++++++++++++ spec/schemas/flow_framework.errors.yaml | 98 +++++++++ tests/default/flow_framework/workflow.yaml | 59 ++++++ 6 files changed, 512 insertions(+), 1 deletion(-) create mode 100644 spec/namespaces/flow_framework.yaml create mode 100644 spec/schemas/flow_framework.common.yaml create mode 100644 spec/schemas/flow_framework.errors.yaml create mode 100644 tests/default/flow_framework/workflow.yaml diff --git a/.cspell b/.cspell index 3656f245c..518d11f61 100644 --- a/.cspell +++ b/.cspell @@ -191,4 +191,5 @@ urldecode vectory whoamiprotected wordnet -Yrtsd \ No newline at end of file +Yrtsd +reprovision \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d253eb39..a961108f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added 404 responses to `/_alias/{name}` and `/{index}/_alias/{name}` ([#519](https://github.com/opensearch-project/opensearch-api-specification/pull/519)) - Added `asynchronous_search` ([#525](https://github.com/opensearch-project/opensearch-api-specification/pull/525)) - Added `DELETE /_plugins/_ml/tasks/{task_id}` ([#530](https://github.com/opensearch-project/opensearch-api-specification/pull/530)) +- Added `/_plugins/_flow_framework` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) - Added `AwarenessAttributeStats` to `/_cluster/health` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cache_reserved_in_bytes` to `ClusterFileSystem` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cluster_manager` to `ClusterNodeCount` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml new file mode 100644 index 000000000..eff3c923a --- /dev/null +++ b/spec/namespaces/flow_framework.yaml @@ -0,0 +1,225 @@ +openapi: 3.1.0 +info: + title: OpenSearch Flow Framework API + description: OpenSearch Flow Framework API. + version: 1.0.0 +paths: + /_plugins/_flow_framework/workflow: + post: + operationId: flow_framework.create.0 + x-operation-group: flow_framework.create + x-version-added: '2.12' + description: Create a workflow. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.create::query.provision' + - $ref: '#/components/parameters/flow_framework.create::query.reprovision' + - $ref: '#/components/parameters/flow_framework.create::query.use_case' + - $ref: '#/components/parameters/flow_framework.create::query.validation' + requestBody: + $ref: '#/components/requestBodies/flow_framework.create' + responses: + '201': + $ref: '#/components/responses/flow_framework.create@201' + '403': + $ref: '#/components/responses/flow_framework.create@403' + '400': + $ref: '#/components/responses/flow_framework.create@400' + /_plugins/_flow_framework/workflow/{workflow_id}: + get: + operationId: flow_framework.get.0 + x-operation-group: flow_framework.get + x-version-added: '2.12' + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/get-workflow/ + description: Get a workflow. + parameters: + - $ref: '#/components/parameters/flow_framework.get::path.workflow_id' + responses: + '200': + $ref: '#/components/responses/flow_framework.get@200' + '404': + $ref: '#/components/responses/flow_framework.get@404' + put: + operationId: flow_framework.update.0 + x-operation-group: flow_framework.update + x-version-added: '2.12' + description: Update a workflow. You can only update a complete workflow if it has not yet been provisioned. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.update::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.update::query.update_fields' + requestBody: + $ref: '#/components/requestBodies/flow_framework.update' + responses: + '201': + $ref: '#/components/responses/flow_framework.update@201' + description: It returns the workflow_id + '404': + $ref: '#/components/responses/flow_framework.update@404' + delete: + operationId: flow_framework.delete.0 + x-operation-group: flow_framework.delete + x-version-added: '2.12' + description: Delete a workflow. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/delete-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.delete::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.delete::query.clear_status' + responses: + '200': + $ref: '#/components/responses/flow_framework.delete@200' + '400': + $ref: '#/components/responses/flow_framework.delete@400' + '403': + $ref: '#/components/responses/flow_framework.delete@403' + '404': + $ref: '#/components/responses/flow_framework.delete@404' +components: + parameters: + flow_framework.get::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + type: string + flow_framework.update::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + type: string + flow_framework.delete::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + type: string + flow_framework.create::query.provision: + in: query + name: provision + schema: + description: Provision the workflow as part of the request + type: boolean + default: false + flow_framework.create::query.reprovision: + x-version-added: '2.17' + in: query + name: reprovision + schema: + type: boolean + default: false + flow_framework.create::query.use_case: + description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/workflow-templates/#supported-workflow-templates + in: query + name: use_case + schema: + type: string + flow_framework.create::query.validation: + in: query + name: validation + schema: + description: Validate the workflow. Valid values are all (validate the template) and none (do not validate the template). Default is all. + type: string + default: all + flow_framework.update::query.update_fields: + in: query + name: update_fields + schema: + description: Update only the fields included in the request body. + type: boolean + default: false + flow_framework.delete::query.clear_status: + in: query + name: clear_status + schema: + description: Delete the workflow state (without deprovisioning resources) after deleting the template. OpenSearch deletes the workflow state only if the provisioning status is not IN_PROGRESS. Default is false. + type: boolean + default: false + requestBodies: + flow_framework.create: + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkCreate' + flow_framework.update: + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkUpdate' + responses: + flow_framework.create@201: + content: + application/json: + schema: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow to be updated. Required for the PUT method. + required: + - workflow_id + flow_framework.create@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.create@400: + description: Bad Request - Multiple possible reasons + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + flow_framework.update@201: + content: + application/json: + schema: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow to be updated. Required for the PUT method. + required: + - workflow_id + flow_framework.delete@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse' + flow_framework.get@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkGetResponse' + flow_framework.update@404: + description: Template Not Found Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.delete@400: + description: Work Flow Id Null Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' + flow_framework.delete@403: + description: Flow Framework API Disabled Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.delete@404: + description: Template Not Found Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.get@404: + description: Template Not Found Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' \ No newline at end of file diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml new file mode 100644 index 000000000..6ff3cda19 --- /dev/null +++ b/spec/schemas/flow_framework.common.yaml @@ -0,0 +1,127 @@ +openapi: 3.0.3 +info: + title: Schemas of flow_framework.common category + description: Schemas of flow_framework.common category + version: 1.0.0 +paths: { } +components: + schemas: + FlowFrameworkCreate: + type: object + properties: + name: + type: string + description: The name of the workflow. + description: + type: string + description: A description of the workflow. + use_case: + type: string + description: A use case, which can be used with the Search Workflow API to find related workflows. + version: + $ref: '#/components/schemas/version' + workflows: + type: object + description: A map of workflows. Presently, only the provision key is supported. The value for the workflow key is a key-value map that includes fields for user_params and lists of nodes and edges. + required: + - name + FlowFrameworkUpdate: + type: object + properties: + name: + type: string + description: The name of the workflow. + description: + type: string + description: A description of the workflow. + use_case: + type: string + description: A use case, which can be used with the Search Workflow API to find related workflows. + version: + $ref: '#/components/schemas/version' + minProperties: 1 + FlowFrameworkDeleteResponse: + type: object + properties: + _index: + type: string + description: The name of the index where the document was stored. + _id: + type: string + description: The unique identifier of the deleted document. + _version: + type: integer + description: The version of the document after the deletion. + result: + type: string + description: The result of the deletion operation. + enum: [deleted, not_found] + _shards: + $ref: '#/components/schemas/shards' + _seq_no: + type: integer + description: The sequence number assigned to the document after the deletion. + _primary_term: + type: integer + description: The primary term assigned to the document after the deletion. + FlowFrameworkGetResponse: + type: object + properties: + name: + type: string + description: + type: string + use_case: + type: string + workflows: + type: object + user: + $ref: '#/components/schemas/user' + created_time: + type: integer + last_updated_time: + type: integer + version: + type: object + description: A key-value map with two fields, template, which identifies the template version, and compatibility, which identifies a list of minimum required OpenSearch versions. + properties: + template: + type: string + description: The template version of the workflow. + compatibility: + type: array + items: + type: string + description: A list of minimum required OpenSearch versions. + shards: + type: object + properties: + total: + type: integer + description: The total number of shards involved in the deletion. + successful: + type: integer + description: The number of shards where the deletion was successful. + failed: + type: integer + description: The number of shards where the deletion failed. + user: + type: object + properties: + name: + type: string + backend_roles: + type: array + items: + type: string + roles: + type: array + items: + type: string + custom_attribute_names: + type: array + items: + type: string + user_requested_tenant: + type: string + nullable: true \ No newline at end of file diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml new file mode 100644 index 000000000..0224b1cba --- /dev/null +++ b/spec/schemas/flow_framework.errors.yaml @@ -0,0 +1,98 @@ +openapi: 3.1.0 +info: + title: Schemas of flow_framework._errors category + description: Schemas of flow_framework._errors category + version: 1.0.0 +paths: {} +components: + schemas: + FlowFrameworkAPIDisabledError: + content: + application/json: + type: object + properties: + message: + type: string + example: This API is disabled. To enable it, set [flow_framework.enabled] to true. + status: + type: integer + BadRequestError: + content: + application/json: + type: object + properties: + message: + type: string + example: Only the parameters [param1, param2] are permitted unless the provision parameter is set to true. + status: + type: integer + ConflictError: + content: + application/json: + type: object + properties: + message: + type: string + example: You cannot use both the 'provision_workflow' and 'update_workflow_fields' parameters in the same request. + status: + type: integer + MissingParameterError: + content: + application/json: + type: object + properties: + message: + type: string + example: You cannot use the 'reprovision_workflow' parameter to create a new template. + status: + type: integer + ParameterConflictError: + content: + application/json: + type: object + properties: + message: + type: string + example: You cannot use the 'reprovision_workflow' and 'use_case' parameters in the same request. + status: + type: integer + WorkFlowIdNullError: + content: + application/json: + type: object + properties: + message: + type: string + example: Workflow ID can not be null + status: + type: integer + WorkflowSaveError: + content: + application/json: + type: object + properties: + message: + type: string + example: Failed to save workflow state + status: + type: integer + MaxWorkflowsLimitError: + content: + application/json: + type: object + properties: + message: + type: string + example: Maximum workflows limit reached 50 + code: + type: integer + TemplateNotFoundError: + content: + application/json: + type: object + properties: + message: + type: string + example: Failed to retrieve template (12345) from global context. + code: + type: integer \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml new file mode 100644 index 000000000..53da81c66 --- /dev/null +++ b/tests/default/flow_framework/workflow.yaml @@ -0,0 +1,59 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework endpoints. +version: '>= 2.12' +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Create workflow. + id: create_flow_framework + path: /_plugins/_flow_framework/workflow + method: POST + request: + payload: + name: test_create_work_flow + response: + status: 201 + output: + test_workflow_id: payload.workflow_id + - synopsis: Get workflow. + id: get_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Update workflow. + id: get_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + request: + payload: + name: test_update_work_flow + response: + status: 201 + - synopsis: Update workflow with a non-existent workflow ID. + id: get_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: test + request: + payload: + name: test_update_work_flow + response: + status: 404 + - synopsis: Delete workflow. + id: delete_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 \ No newline at end of file From 5cd57c41ea790b1d4654401828e225afbf10cb58 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 9 Sep 2024 10:56:53 -0400 Subject: [PATCH 03/73] Update spec and add tests for xy, geo shape, and geo bounding box query (#531) * Adds xy shape query spec and test Signed-off-by: Fanit Kolchina * Update spec and add tests for geo bounding box and geo shape Signed-off-by: Fanit Kolchina * Address linter errors Signed-off-by: Fanit Kolchina * Add version added to xy field types and query Signed-off-by: Fanit Kolchina * Fix lint errors Signed-off-by: Fanit Kolchina * Remove ShapeQuery from spec and make list ordered Signed-off-by: Fanit Kolchina * Extract objects into refs Signed-off-by: Fanit Kolchina * Extract last object and add changelog entry Signed-off-by: Fanit Kolchina * Implement code review comments Signed-off-by: Fanit Kolchina * Fix lint errors Signed-off-by: Fanit Kolchina * Apply suggestions from code review Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> --------- Signed-off-by: Fanit Kolchina Signed-off-by: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 3 + spec/schemas/_common.mapping.yaml | 20 +++-- spec/schemas/_common.query_dsl.yaml | 74 ++++++++++++++----- spec/schemas/_common.yaml | 26 +++++++ .../_core/search/query/geo_bounding_box.yaml | 60 +++++++++++++++ .../_core/search/query/geo_shape_point.yaml | 52 +++++++++++++ .../_core/search/query/geo_shape_shape.yaml | 58 +++++++++++++++ .../_core/search/query/xy_shape_point.yaml | 53 +++++++++++++ .../_core/search/query/xy_shape_shape.yaml | 58 +++++++++++++++ 9 files changed, 380 insertions(+), 24 deletions(-) create mode 100644 tests/default/_core/search/query/geo_bounding_box.yaml create mode 100644 tests/default/_core/search/query/geo_shape_point.yaml create mode 100644 tests/default/_core/search/query/geo_shape_shape.yaml create mode 100644 tests/default/_core/search/query/xy_shape_point.yaml create mode 100644 tests/default/_core/search/query/xy_shape_shape.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a961108f6..942d32ecf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added request and response schemas for `/_cluster/routing/awareness/{attribute}/weights` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) +- Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) ### Changed @@ -114,6 +115,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Removed - Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358)) +- Removed `shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) ### Fixed @@ -145,6 +147,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed content-type for `/hot_threads` ([#543](https://github.com/opensearch-project/opensearch-api-specification/pull/543)) - Fixed `/_cluster/settings` returning flat results ([#545](https://github.com/opensearch-project/opensearch-api-specification/pull/545)) - Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) +- Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) ### Security diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index ef7a31a31..14edfcebe 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -191,8 +191,8 @@ components: - $ref: '#/components/schemas/TokenCountProperty' - $ref: '#/components/schemas/GeoPointProperty' - $ref: '#/components/schemas/GeoShapeProperty' - - $ref: '#/components/schemas/PointProperty' - - $ref: '#/components/schemas/ShapeProperty' + - $ref: '#/components/schemas/XyPointProperty' + - $ref: '#/components/schemas/XyShapeProperty' - $ref: '#/components/schemas/ByteNumberProperty' - $ref: '#/components/schemas/DoubleNumberProperty' - $ref: '#/components/schemas/FloatNumberProperty' @@ -913,6 +913,10 @@ components: $ref: '#/components/schemas/GeoOrientation' strategy: $ref: '#/components/schemas/GeoStrategy' + deprecated: true + distance_error_pct: + type: number + deprecated: true type: type: string enum: @@ -929,7 +933,8 @@ components: enum: - recursive - term - PointProperty: + XyPointProperty: + # x-version-added: 2.4 allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' - type: object @@ -939,14 +944,15 @@ components: ignore_z_value: type: boolean null_value: - type: string + $ref: '_common.yaml#/components/schemas/XyLocation' type: type: string enum: - - point + - xy_point required: - type - ShapeProperty: + XyShapeProperty: + # x-version-added: 2.4 allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' - type: object @@ -962,7 +968,7 @@ components: type: type: string enum: - - shape + - xy_shape required: - type ByteNumberProperty: diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index bb2c44c7d..7bb1b4a29 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -153,8 +153,6 @@ components: $ref: '#/components/schemas/ScriptQuery' script_score: $ref: '#/components/schemas/ScriptScoreQuery' - shape: - $ref: '#/components/schemas/ShapeQuery' simple_query_string: $ref: '#/components/schemas/SimpleQueryStringQuery' span_containing: @@ -209,6 +207,8 @@ components: $ref: '#/components/schemas/TextExpansionQuery' minProperties: 1 maxProperties: 1 + type: + $ref: '#/components/schemas/TypeQuery' weighted_tokens: description: Supports returning text_expansion query results by sending in precomputed tokens with the query. type: object @@ -225,8 +225,8 @@ components: maxProperties: 1 wrapper: $ref: '#/components/schemas/WrapperQuery' - type: - $ref: '#/components/schemas/TypeQuery' + xy_shape: + $ref: '#/components/schemas/XyShapeQuery' minProperties: 1 maxProperties: 1 BoolQuery: @@ -629,6 +629,9 @@ components: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: $ref: '#/components/schemas/IgnoreUnmapped' + additionalProperties: + $ref: '_common.yaml#/components/schemas/GeoBounds' + minProperties: 1 GeoExecution: type: string enum: @@ -667,6 +670,22 @@ components: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: $ref: '#/components/schemas/IgnoreUnmapped' + GeoShape: + type: object + properties: + type: + type: string + coordinates: + type: array + GeoShapeField: + type: object + properties: + shape: + $ref: '#/components/schemas/GeoShape' + relation: + $ref: '_common.yaml#/components/schemas/GeoShapeRelation' + required: + - shape GeoShapeQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -674,6 +693,9 @@ components: properties: ignore_unmapped: $ref: '#/components/schemas/IgnoreUnmapped' + additionalProperties: + $ref: '#/components/schemas/GeoShapeField' + minProperties: 1 HasChildQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -1626,13 +1648,6 @@ components: required: - query - script - ShapeQuery: - allOf: - - $ref: '#/components/schemas/QueryBase' - - type: object - properties: - ignore_unmapped: - $ref: '#/components/schemas/IgnoreUnmapped' SimpleQueryStringQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -1918,6 +1933,15 @@ components: only_score_pruned_tokens: description: Whether to only score pruned tokens, vs only scoring kept tokens. type: boolean + TypeQuery: + allOf: + - $ref: '#/components/schemas/QueryBase' + - type: object + properties: + value: + type: string + required: + - value WeightedTokensQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -1960,12 +1984,28 @@ components: type: string required: - query - TypeQuery: + XyShape: + type: object + properties: + type: + type: string + coordinates: + type: array + XyShapeField: + type: object + properties: + shape: + $ref: '#/components/schemas/XyShape' + relation: + $ref: '_common.yaml#/components/schemas/GeoShapeRelation' + required: + - shape + XyShapeQuery: + x-version-added: 2.4 allOf: - $ref: '#/components/schemas/QueryBase' - type: object - properties: - value: - type: string - required: - - value + additionalProperties: + $ref: '#/components/schemas/XyShapeField' + minProperties: 1 + maxProperties: 1 \ No newline at end of file diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 7e55ff907..845204b3a 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -133,6 +133,32 @@ components: type: string required: - wkt + XyLocation: + x-version-added: 2.4 + description: |- + A two-dimensional Cartesian point specified by x and y coordinates. It can be represented in various ways: + - as a `{x, y}` object + - as a `[x, y]` array + - as a string in `"x, y"` or WKT point formats. + oneOf: + - $ref: '#/components/schemas/XyCartesianCoordinates' + - type: array + items: + type: number + - type: string + XyCartesianCoordinates: + x-version-added: '2.4' + type: object + properties: + x: + description: The x coordinate. + type: number + y: + description: The y coordinate. + type: number + required: + - x + - y EpochTimeUnitMillis: allOf: - $ref: '#/components/schemas/UnitMillis' diff --git a/tests/default/_core/search/query/geo_bounding_box.yaml b/tests/default/_core/search/query/geo_bounding_box.yaml new file mode 100644 index 000000000..1bdc7f605 --- /dev/null +++ b/tests/default/_core/search/query/geo_bounding_box.yaml @@ -0,0 +1,60 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with geo_bounding_box query. +prologues: + - path: /point_index + method: PUT + request: + payload: + mappings: + properties: + field: + type: geo_point + - path: /point_index/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + field: + lat: 74 + lon: 40 + status: [201] +epilogues: + - path: /point_index + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search point_index for documents whose point objects are within the specified bounding box. + path: /{index}/_search + parameters: + index: point_index + method: GET + request: + payload: + query: + geo_bounding_box: + field: + top: 75 + left: 28 + bottom: 73 + right: 41 + validation_method: strict + type: memory + ignore_unmapped: false + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 1 + hits: + - _index: point_index + _score: 1 + _source: + field: + lat: 74 + lon: 40 \ No newline at end of file diff --git a/tests/default/_core/search/query/geo_shape_point.yaml b/tests/default/_core/search/query/geo_shape_point.yaml new file mode 100644 index 000000000..d55748f95 --- /dev/null +++ b/tests/default/_core/search/query/geo_shape_point.yaml @@ -0,0 +1,52 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with geo_shape query. +prologues: + - path: /cinemas + method: PUT + request: + payload: + mappings: + properties: + location: + type: geo_point + - path: /cinemas/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + location: [0.5, 3] + status: [201] +epilogues: + - path: /cinemas + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search cinemas for documents whose point objects are within the specified envelope. + path: /{index}/_search + parameters: + index: cinemas + method: GET + request: + payload: + query: + geo_shape: + location: + shape: + type: envelope + coordinates: [[0, 6], [1, 2]] + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 0 + hits: + - _index: cinemas + _score: 0 + _source: + location: [0.5, 3] \ No newline at end of file diff --git a/tests/default/_core/search/query/geo_shape_shape.yaml b/tests/default/_core/search/query/geo_shape_shape.yaml new file mode 100644 index 000000000..224805a60 --- /dev/null +++ b/tests/default/_core/search/query/geo_shape_shape.yaml @@ -0,0 +1,58 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with geo_shape query. +prologues: + - path: /cinemas + method: PUT + request: + payload: + mappings: + properties: + location: + type: geo_shape + - path: /cinemas/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + location: + type: point + coordinates: [0.5, 3] + status: [201] +epilogues: + - path: /cinemas + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search cinemas for documents whose point objects are within the specified envelope. + path: /{index}/_search + parameters: + index: cinemas + method: GET + request: + payload: + query: + geo_shape: + location: + shape: + type: envelope + coordinates: [[0, 6], [1, 2]] + relation: within + ignore_unmapped: true + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 0 + hits: + - _index: cinemas + _score: 0 + _source: + location: + type: point + coordinates: [0.5, 3] diff --git a/tests/default/_core/search/query/xy_shape_point.yaml b/tests/default/_core/search/query/xy_shape_point.yaml new file mode 100644 index 000000000..2e3a12995 --- /dev/null +++ b/tests/default/_core/search/query/xy_shape_point.yaml @@ -0,0 +1,53 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with xy query. +version: '>= 2.4' +prologues: + - path: /point_index + method: PUT + request: + payload: + mappings: + properties: + field: + type: xy_point + - path: /point_index/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + field: [0.5, 3] + status: [201] +epilogues: + - path: /point_index + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search point_index for documents whose point objects are within the specified envelope. + path: /{index}/_search + parameters: + index: point_index + method: GET + request: + payload: + query: + xy_shape: + field: + shape: + type: envelope + coordinates: [[0, 6], [1, 2]] + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 0 + hits: + - _index: point_index + _score: 0 + _source: + field: [0.5, 3] \ No newline at end of file diff --git a/tests/default/_core/search/query/xy_shape_shape.yaml b/tests/default/_core/search/query/xy_shape_shape.yaml new file mode 100644 index 000000000..74f87eba0 --- /dev/null +++ b/tests/default/_core/search/query/xy_shape_shape.yaml @@ -0,0 +1,58 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with xy query. +version: '>= 2.4' +prologues: + - path: /shape_index + method: PUT + request: + payload: + mappings: + properties: + field: + type: xy_shape + - path: /shape_index/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + field: + type: point + coordinates: [0.5, 3] + status: [201] +epilogues: + - path: /shape_index + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search shape_index for documents whose point objects are within the specified envelope. + path: /{index}/_search + parameters: + index: shape_index + method: GET + request: + payload: + query: + xy_shape: + field: + shape: + type: envelope + coordinates: [[0, 6], [1, 2]] + relation: within + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 0 + hits: + - _index: shape_index + _score: 0 + _source: + field: + type: point + coordinates: [0.5, 3] \ No newline at end of file From 49f7d99bde265436506d66407314ae10031f640c Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:40:11 -0400 Subject: [PATCH 04/73] Fix fuzzy query rewrite parameter and add test (#561) Co-authored-by: Theo N. Truong --- CHANGELOG.md | 1 + spec/schemas/_common.yaml | 7 +++ tests/default/_core/search/query/fuzzy.yaml | 56 +++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/default/_core/search/query/fuzzy.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 942d32ecf..eb5cca8ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed content-type for `/hot_threads` ([#543](https://github.com/opensearch-project/opensearch-api-specification/pull/543)) - Fixed `/_cluster/settings` returning flat results ([#545](https://github.com/opensearch-project/opensearch-api-specification/pull/545)) - Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) +- Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561)) - Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) ### Security diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 845204b3a..3c7c8c6aa 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -460,6 +460,13 @@ components: - id MultiTermQueryRewrite: type: string + enum: + - constant_score + - constant_score_boolean + - scoring_boolean + - top_terms_N + - top_terms_blended_freqs_N + - top_terms_boost_N Fuzziness: oneOf: - type: string diff --git a/tests/default/_core/search/query/fuzzy.yaml b/tests/default/_core/search/query/fuzzy.yaml new file mode 100644 index 000000000..7032089da --- /dev/null +++ b/tests/default/_core/search/query/fuzzy.yaml @@ -0,0 +1,56 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with fuzzy query. +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + title: + type: text + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + title: Brave + status: [201] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search for documents using a fuzzy query. + path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + query: + fuzzy: + title: + value: bake + fuzziness: 2 + max_expansions: 40 + prefix_length: 0 + transpositions: true + rewrite: constant_score + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + max_score: 1 + hits: + - _index: movies + _score: 1 + _source: + title: Brave + From d279f4a50b3c88be63a8bf3af0caa71244fdc327 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 9 Sep 2024 11:40:26 -0400 Subject: [PATCH 05/73] Fix geo distance query and its test (#560) Co-authored-by: Theo N. Truong --- CHANGELOG.md | 1 + spec/schemas/_common.query_dsl.yaml | 6 +++--- .../search/{ => query}/geo_distance.yaml | 20 +++++++++---------- 3 files changed, 14 insertions(+), 13 deletions(-) rename tests/default/_core/search/{ => query}/geo_distance.yaml (80%) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb5cca8ee..c48b84eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed query DSL `neural` field `query_image` set `contentEncoding` and `model_id` as optional ([#512](https://github.com/opensearch-project/opensearch-api-specification/pull/512)) - Fixed `knn` query specification ([#538](https://github.com/opensearch-project/opensearch-api-specification/pull/538)) - Fixed content-type for `/hot_threads` ([#543](https://github.com/opensearch-project/opensearch-api-specification/pull/543)) +- Fixed `geo_distance` query spec ([#560](https://github.com/opensearch-project/opensearch-api-specification/pull/560)) - Fixed `/_cluster/settings` returning flat results ([#545](https://github.com/opensearch-project/opensearch-api-specification/pull/545)) - Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561)) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 7bb1b4a29..51dead39c 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -656,11 +656,11 @@ components: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: $ref: '#/components/schemas/IgnoreUnmapped' - field: - type: object + additionalProperties: + $ref: '_common.yaml#/components/schemas/GeoLocation' + minProperties: 2 required: - distance - - field GeoPolygonQuery: allOf: - $ref: '#/components/schemas/QueryBase' diff --git a/tests/default/_core/search/geo_distance.yaml b/tests/default/_core/search/query/geo_distance.yaml similarity index 80% rename from tests/default/_core/search/geo_distance.yaml rename to tests/default/_core/search/query/geo_distance.yaml index 3beff6865..21e93d156 100644 --- a/tests/default/_core/search/geo_distance.yaml +++ b/tests/default/_core/search/query/geo_distance.yaml @@ -1,34 +1,34 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with geo_distance query. prologues: - - path: /map + - path: /cinemas method: PUT request: payload: mappings: properties: - field: + location: type: geo_point - - path: /map/_doc/1 + - path: /cinemas/_doc/1 method: POST parameters: refresh: true request: payload: - field: + location: lat: 74 lon: 40.71 status: [201] epilogues: - - path: /map + - path: /cinemas method: DELETE status: [200, 404] chapters: - synopsis: Search for documents whose point objects are within the specified distance from the specified point. path: /{index}/_search parameters: - index: map + index: cinemas method: GET request: payload: @@ -38,7 +38,7 @@ chapters: distance_type: arc validation_method: strict ignore_unmapped: true - field: + location: lat: 73.5 lon: 40.5 response: @@ -51,10 +51,10 @@ chapters: relation: eq max_score: 1 hits: - - _index: map + - _index: cinemas _score: 1 _source: - field: + location: lat: 74 lon: 40.71 From a768a73f8f8ec04b6464be4ac6b50e52c994da25 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:21:19 -0400 Subject: [PATCH 06/73] Rename index and field in geo_bounding_box query test (#562) Signed-off-by: Fanit Kolchina --- .../_core/search/query/geo_bounding_box.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/default/_core/search/query/geo_bounding_box.yaml b/tests/default/_core/search/query/geo_bounding_box.yaml index 1bdc7f605..bee1e0a30 100644 --- a/tests/default/_core/search/query/geo_bounding_box.yaml +++ b/tests/default/_core/search/query/geo_bounding_box.yaml @@ -2,39 +2,39 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with geo_bounding_box query. prologues: - - path: /point_index + - path: /cinemas method: PUT request: payload: mappings: properties: - field: + location: type: geo_point - - path: /point_index/_doc/1 + - path: /cinemas/_doc/1 method: POST parameters: refresh: true request: payload: - field: + location: lat: 74 lon: 40 status: [201] epilogues: - - path: /point_index + - path: /cinemas method: DELETE status: [200, 404] chapters: - - synopsis: Search point_index for documents whose point objects are within the specified bounding box. + - synopsis: Search cinemas for documents whose point objects are within the specified bounding box. path: /{index}/_search parameters: - index: point_index + index: cinemas method: GET request: payload: query: geo_bounding_box: - field: + location: top: 75 left: 28 bottom: 73 @@ -52,9 +52,9 @@ chapters: relation: eq max_score: 1 hits: - - _index: point_index + - _index: cinemas _score: 1 _source: - field: + location: lat: 74 lon: 40 \ No newline at end of file From 29185f4948f4b038e23b3d6457f1f54fa5e7b0aa Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 9 Sep 2024 17:55:30 -0400 Subject: [PATCH 07/73] Separated merger and tester fixtures, fixed leftoever nulls. (#559) * Separated merger and tester fixtures. Signed-off-by: dblock * Fix: delete leftoever undefined array items. Signed-off-by: dblock * Re-add x-version-added for geo types. Signed-off-by: dblock --------- Signed-off-by: dblock --- spec/schemas/_common.mapping.yaml | 4 +- spec/schemas/_common.query_dsl.yaml | 2 +- spec/schemas/_common.yaml | 2 +- tools/src/helpers.ts | 4 + tools/tests/helpers.test.ts | 19 +++ tools/tests/merger/OpenApiMerger.test.ts | 4 +- .../merger/OpenApiVersionExtractor.test.ts | 12 +- .../{ => opensearch}/expected_1.3.yaml | 10 ++ .../{ => opensearch}/expected_2.0.yaml | 7 ++ .../merger/{ => animals}/expected.yaml | 0 .../animals}/_global_parameters.yaml | 0 .../{spec => specs/animals}/_info.yaml | 0 .../animals}/_superseded_operations.yaml | 0 .../animals}/namespaces/indices.yaml | 0 .../animals}/namespaces/shelter.yaml | 0 .../animals}/schemas/actions.yaml | 0 .../animals}/schemas/animals.yaml | 0 .../specs/opensearch/_global_parameters.yaml | 4 + .../fixtures/specs/opensearch/_info.yaml | 6 + .../opensearch/_superseded_operations.yaml | 1 + .../namespaces/cluster_manager.yaml | 17 +++ .../specs/opensearch/namespaces/index.yaml | 117 ++++++++++++++++++ .../specs/opensearch/namespaces/nodes.yaml | 90 ++++++++++++++ .../specs/opensearch/schemas/_common.yaml | 18 +++ 24 files changed, 305 insertions(+), 12 deletions(-) rename tools/tests/merger/fixtures/extractor/{ => opensearch}/expected_1.3.yaml (90%) rename tools/tests/merger/fixtures/extractor/{ => opensearch}/expected_2.0.yaml (94%) rename tools/tests/merger/fixtures/merger/{ => animals}/expected.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/_global_parameters.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/_info.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/_superseded_operations.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/namespaces/indices.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/namespaces/shelter.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/schemas/actions.yaml (100%) rename tools/tests/merger/fixtures/{spec => specs/animals}/schemas/animals.yaml (100%) create mode 100644 tools/tests/merger/fixtures/specs/opensearch/_global_parameters.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/_info.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/_superseded_operations.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/namespaces/cluster_manager.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/namespaces/index.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/namespaces/nodes.yaml create mode 100644 tools/tests/merger/fixtures/specs/opensearch/schemas/_common.yaml diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 14edfcebe..a26c15e61 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -934,7 +934,7 @@ components: - recursive - term XyPointProperty: - # x-version-added: 2.4 + x-version-added: '2.4' allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' - type: object @@ -952,7 +952,7 @@ components: required: - type XyShapeProperty: - # x-version-added: 2.4 + x-version-added: '2.4' allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' - type: object diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 51dead39c..c532ef260 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -2001,7 +2001,7 @@ components: required: - shape XyShapeQuery: - x-version-added: 2.4 + x-version-added: '2.4' allOf: - $ref: '#/components/schemas/QueryBase' - type: object diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 3c7c8c6aa..22c70dbfa 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -134,7 +134,7 @@ components: required: - wkt XyLocation: - x-version-added: 2.4 + x-version-added: '2.4' description: |- A two-dimensional Cartesian point specified by x and y coordinates. It can be represented in various ways: - as a `{x, y}` object diff --git a/tools/src/helpers.ts b/tools/src/helpers.ts index 97cee98f3..4f1b346ef 100644 --- a/tools/src/helpers.ts +++ b/tools/src/helpers.ts @@ -67,11 +67,15 @@ export function sort_array_by_keys (values: any[], priorities: string[] = []): s export function delete_matching_keys(obj: any, condition: (obj: any) => boolean): void { for (const key in obj) { var item = obj[key] + if (_.isObject(item)) { if (condition(item)) { delete obj[key] } else { delete_matching_keys(item, condition) + if (_.isArray(item)) { + obj[key] = _.compact(item) + } } } } diff --git a/tools/tests/helpers.test.ts b/tools/tests/helpers.test.ts index 54773d226..f91dde573 100644 --- a/tools/tests/helpers.test.ts +++ b/tools/tests/helpers.test.ts @@ -122,6 +122,25 @@ describe('helpers', () => { expect(obj).toStrictEqual({ foo: [{}] }) }) }) + + describe('an object with an array where some items are removed', () => { + var obj: object + + beforeEach(() => { + obj = { + foo: [ + { value: 1 }, + { value: 1 }, + { value: 2 } + ] + } + }) + + test('removes matching values', () => { + delete_matching_keys(obj, (_item: any) => _item.value == 1) + expect(obj).toStrictEqual({ foo: [{ value: 2 }] }) + }) + }) }) describe('find_refs', () => { diff --git a/tools/tests/merger/OpenApiMerger.test.ts b/tools/tests/merger/OpenApiMerger.test.ts index 057866dd2..d0beb2bb0 100644 --- a/tools/tests/merger/OpenApiMerger.test.ts +++ b/tools/tests/merger/OpenApiMerger.test.ts @@ -16,7 +16,7 @@ describe('OpenApiMerger', () => { describe('defaults', () => { beforeEach(() => { - merger = new OpenApiMerger('./tools/tests/merger/fixtures/spec/') + merger = new OpenApiMerger('./tools/tests/merger/fixtures/specs/animals') }) describe('merge()', () => { @@ -45,7 +45,7 @@ describe('OpenApiMerger', () => { test('writes a spec', () => { merger.write_to(filename) - expect(fs.readFileSync('./tools/tests/merger/fixtures/merger/expected.yaml', 'utf8')) + expect(fs.readFileSync('./tools/tests/merger/fixtures/merger/animals/expected.yaml', 'utf8')) .toEqual(fs.readFileSync(filename, 'utf8')) }) }) diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index ca3e14eb0..ab0dc97e9 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -14,7 +14,7 @@ import fs from 'fs' import tmp from 'tmp' describe('extract() from a merged API spec', () => { - const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') + const merger = new OpenApiMerger('tools/tests/merger/fixtures/specs/opensearch') describe('1.3', () => { const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') @@ -35,7 +35,7 @@ describe('extract() from a merged API spec', () => { test('writes a spec', () => { extractor.write_to(filename) - expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/expected_1.3.yaml', 'utf8')) + expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml', 'utf8')) .toEqual(fs.readFileSync(filename, 'utf8')) }) }) @@ -43,7 +43,7 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' + '200', '201', '404', '500', '503', 'removed-2.0', 'removed-2.0-refs', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' ]) }) }) @@ -54,7 +54,7 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless' + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0-refs', 'distributed-excluded-amazon-serverless' ]) }) @@ -74,7 +74,7 @@ describe('extract() from a merged API spec', () => { test('writes a spec', () => { extractor.write_to(filename) - expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/expected_2.0.yaml', 'utf8')) + expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml', 'utf8')) .toEqual(fs.readFileSync(filename, 'utf8')) }) }) @@ -86,7 +86,7 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless' + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0-refs', 'added-2.1', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml similarity index 90% rename from tools/tests/merger/fixtures/extractor/expected_1.3.yaml rename to tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml index a4497ccbb..24d08aa4d 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml @@ -33,6 +33,8 @@ paths: removed-2.0: $ref: '#/components/responses/info@removed-2.0' x-version-removed: '2.0' + removed-2.0-refs: + $ref: '#/components/responses/info@removed-2.0-refs' added-1.3-removed-2.0: $ref: '#/components/responses/info@added-1.3-removed-2.0' distributed-excluded-amazon-serverless: @@ -114,6 +116,12 @@ components: description: Distributed in opensearch.org, AOS and AOSS. info@removed-2.0: description: Removed in 2.0 via attribute next to ref. + info@removed-2.0-refs: + description: One of the ref values removed in 2.0. + schema: + oneOf: + - $ref: '#/components/schemas/_common:Type' + - $ref: '#/components/schemas/_common:OldId' nodes.info@200: description: All nodes. content: @@ -121,6 +129,8 @@ components: schema: type: object schemas: + _common:OldId: + type: string _common:Type: type: string x-version-removed: '2.0' diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml similarity index 94% rename from tools/tests/merger/fixtures/extractor/expected_2.0.yaml rename to tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml index c684b7af9..a824f4787 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml @@ -57,6 +57,8 @@ paths: added-2.0: $ref: '#/components/responses/info@added-2.0' x-version-added: '2.0' + removed-2.0-refs: + $ref: '#/components/responses/info@removed-2.0-refs' distributed-excluded-amazon-serverless: $ref: '#/components/responses/info@distributed-all' x-distributions-excluded: @@ -150,6 +152,11 @@ components: description: Added in 2.0 via attribute next to ref. info@distributed-all: description: Distributed in opensearch.org, AOS and AOSS. + info@removed-2.0-refs: + description: One of the ref values removed in 2.0. + schema: + oneOf: + - $ref: '#/components/schemas/_common:OldId' nodes.info@200: description: All nodes. content: diff --git a/tools/tests/merger/fixtures/merger/expected.yaml b/tools/tests/merger/fixtures/merger/animals/expected.yaml similarity index 100% rename from tools/tests/merger/fixtures/merger/expected.yaml rename to tools/tests/merger/fixtures/merger/animals/expected.yaml diff --git a/tools/tests/merger/fixtures/spec/_global_parameters.yaml b/tools/tests/merger/fixtures/specs/animals/_global_parameters.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/_global_parameters.yaml rename to tools/tests/merger/fixtures/specs/animals/_global_parameters.yaml diff --git a/tools/tests/merger/fixtures/spec/_info.yaml b/tools/tests/merger/fixtures/specs/animals/_info.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/_info.yaml rename to tools/tests/merger/fixtures/specs/animals/_info.yaml diff --git a/tools/tests/merger/fixtures/spec/_superseded_operations.yaml b/tools/tests/merger/fixtures/specs/animals/_superseded_operations.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/_superseded_operations.yaml rename to tools/tests/merger/fixtures/specs/animals/_superseded_operations.yaml diff --git a/tools/tests/merger/fixtures/spec/namespaces/indices.yaml b/tools/tests/merger/fixtures/specs/animals/namespaces/indices.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/namespaces/indices.yaml rename to tools/tests/merger/fixtures/specs/animals/namespaces/indices.yaml diff --git a/tools/tests/merger/fixtures/spec/namespaces/shelter.yaml b/tools/tests/merger/fixtures/specs/animals/namespaces/shelter.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/namespaces/shelter.yaml rename to tools/tests/merger/fixtures/specs/animals/namespaces/shelter.yaml diff --git a/tools/tests/merger/fixtures/spec/schemas/actions.yaml b/tools/tests/merger/fixtures/specs/animals/schemas/actions.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/schemas/actions.yaml rename to tools/tests/merger/fixtures/specs/animals/schemas/actions.yaml diff --git a/tools/tests/merger/fixtures/spec/schemas/animals.yaml b/tools/tests/merger/fixtures/specs/animals/schemas/animals.yaml similarity index 100% rename from tools/tests/merger/fixtures/spec/schemas/animals.yaml rename to tools/tests/merger/fixtures/specs/animals/schemas/animals.yaml diff --git a/tools/tests/merger/fixtures/specs/opensearch/_global_parameters.yaml b/tools/tests/merger/fixtures/specs/opensearch/_global_parameters.yaml new file mode 100644 index 000000000..2355a0c60 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/_global_parameters.yaml @@ -0,0 +1,4 @@ +openapi: 3.1.0 +info: + title: '' + version: '' diff --git a/tools/tests/merger/fixtures/specs/opensearch/_info.yaml b/tools/tests/merger/fixtures/specs/opensearch/_info.yaml new file mode 100644 index 000000000..acbf44f19 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/_info.yaml @@ -0,0 +1,6 @@ +$schema: should-be-ignored + +title: OpenSearch API +description: OpenSearch API +version: 1.0.0 +x-api-version: 1.2.3 \ No newline at end of file diff --git a/tools/tests/merger/fixtures/specs/opensearch/_superseded_operations.yaml b/tools/tests/merger/fixtures/specs/opensearch/_superseded_operations.yaml new file mode 100644 index 000000000..a5d5e9940 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/_superseded_operations.yaml @@ -0,0 +1 @@ +$schema: should-be-ignored \ No newline at end of file diff --git a/tools/tests/merger/fixtures/specs/opensearch/namespaces/cluster_manager.yaml b/tools/tests/merger/fixtures/specs/opensearch/namespaces/cluster_manager.yaml new file mode 100644 index 000000000..680169ad6 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/namespaces/cluster_manager.yaml @@ -0,0 +1,17 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + /cluster_manager: + get: + operationId: cluster_manager.0 + x-version-added: '2.0' + post: + operationId: cluster_manager.0 + x-version-added: '2.0' +components: + requestBodies: [] + parameters: [] + responses: [] diff --git a/tools/tests/merger/fixtures/specs/opensearch/namespaces/index.yaml b/tools/tests/merger/fixtures/specs/opensearch/namespaces/index.yaml new file mode 100644 index 000000000..f03058350 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/namespaces/index.yaml @@ -0,0 +1,117 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + /index: + get: + operationId: get.0 + responses: + '200': + $ref: '#/components/responses/info@200' + '201': + $ref: '#/components/responses/info@201' + '404': + $ref: '#/components/responses/info@404' + added-2.0: + $ref: '#/components/responses/info@added-2.0' + x-version-added: '2.0' + removed-2.0: + $ref: '#/components/responses/info@removed-2.0' + x-version-removed: '2.0' + removed-2.0-refs: + $ref: '#/components/responses/info@removed-2.0-refs' + added-1.3-removed-2.0: + $ref: '#/components/responses/info@added-1.3-removed-2.0' + added-2.1: + $ref: '#/components/responses/info@added-2.1' + '500': + $ref: '#/components/responses/info@500' + '503': + $ref: '#/components/responses/info@503' + distributed-included-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions-included: + - amazon-managed + - amazon-serverless + - opensearch.org + distributed-included-amazon-managed: + $ref: '#/components/responses/info@distributed-amazon-managed' + x-distributions-included: + - amazon-managed + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless +components: + responses: + info@200: + content: + application/json: + schema: + type: object + properties: + _type: + $ref: '../schemas/_common.yaml#/components/schemas/Type' + tagline: + type: string + required: + - tagline + info@201: + content: + application/json: + schema: + type: object + properties: + tagline: + type: string + required: + - tagline + unevaluatedProperties: true + info@404: + content: + application/json: + schema: + type: object + properties: + tagline: + type: string + required: + - tagline + unevaluatedProperties: + type: object + info@added-2.0: + description: Added in 2.0 via attribute next to ref. + info@removed-2.0: + description: Removed in 2.0 via attribute next to ref. + info@removed-2.0-refs: + description: One of the ref values removed in 2.0. + schema: + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/Type' + - $ref: '../schemas/_common.yaml#/components/schemas/OldId' + info@added-1.3-removed-2.0: + description: Added in 1.3, removed in 2.0 via attribute in response body. + x-version-added: '1.3' + x-version-removed: '2.0' + info@added-2.1: + description: Added in 2.1 via attribute in response body. + x-version-added: '2.1' + info@distributed-amazon-managed: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. + info@500: + content: + application/json: + schema: + type: object + properties: + tagline: + type: string + info@503: + content: + application/json: + schema: + type: object \ No newline at end of file diff --git a/tools/tests/merger/fixtures/specs/opensearch/namespaces/nodes.yaml b/tools/tests/merger/fixtures/specs/opensearch/namespaces/nodes.yaml new file mode 100644 index 000000000..2b39054ea --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/namespaces/nodes.yaml @@ -0,0 +1,90 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + /nodes: + get: + operationId: nodes.0 + responses: + '200': + $ref: '#/components/responses/nodes.info@200' + /_nodes/{id}: + get: + operationId: nodes.info.1 + x-operation-group: nodes.info + x-version-added: '1.0' + description: Returns information about nodes in the cluster. + parameters: + - $ref: '#/components/parameters/nodes.info::path.id' + responses: + '200': + $ref: '#/components/responses/nodes.info@200' + post: + operationId: nodes.info.1 + x-operation-group: nodes.info + x-version-added: '2.0' + description: Returns information about nodes in the cluster. + parameters: + - $ref: '#/components/parameters/nodes.info::path.id' + - $ref: '#/components/parameters/nodes.info::query.flag' + requestBody: + $ref: '#/components/requestBodies/nodes.info' + responses: + '201': + $ref: '#/components/responses/nodes.info@201' + '200': + $ref: '#/components/responses/nodes.info@200' +components: + requestBodies: + nodes.info: + content: + application/json: + schema: + type: object + properties: + _all: + type: boolean + ids: + $ref: '../schemas/_common.yaml#/components/schemas/Ids' + x-version-added: '2.0' + description: Nodes options. + parameters: + nodes.info::path.id: + in: path + name: id + description: Node ID. + required: true + schema: + type: string + nodes.info::query.flag: + in: query + name: flag + description: Flag. + required: false + schema: + type: boolean + default: false + nodes.info::query.new: + in: query + name: new + description: New option added in 2.1. + required: false + x-version-added: '2.1' + schema: + type: array + default: false + responses: + nodes.info@200: + description: All nodes. + content: + application/json: + schema: + type: object + nodes.info@201: + description: All nodes. + content: + application/json: + schema: + type: object diff --git a/tools/tests/merger/fixtures/specs/opensearch/schemas/_common.yaml b/tools/tests/merger/fixtures/specs/opensearch/schemas/_common.yaml new file mode 100644 index 000000000..c3a4e41f3 --- /dev/null +++ b/tools/tests/merger/fixtures/specs/opensearch/schemas/_common.yaml @@ -0,0 +1,18 @@ +openapi: 3.1.0 +info: + title: Schemas of _common category + description: Schemas of _common category + version: 1.0.0 +components: + schemas: + Type: + type: string + x-version-removed: '2.0' + OldId: + type: string + Ids: + oneOf: + - $ref: '#/components/schemas/OldId' + - type: array + items: + $ref: '#/components/schemas/Ids' From 3dcced99f99f8374f0b0f56eaf9ccd813c31f796 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Tue, 10 Sep 2024 12:21:58 +1200 Subject: [PATCH 08/73] Fix terms query schema (#564) * Fix terms query schema Signed-off-by: Thomas Farr * Add changelog Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 2 +- spec/schemas/_common.query_dsl.yaml | 42 +++++++++++++++-------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c48b84eac..0ced75260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `AwarenessAttributeStats` to `/_cluster/health` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cache_reserved_in_bytes` to `ClusterFileSystem` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cluster_manager` to `ClusterNodeCount` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) -- Added support for `query` with `terms` in `_search` ([#546](https://github.com/opensearch-project/opensearch-api-specification/pull/546)) +- Added support for `query` with `terms` in `_search` ([#546](https://github.com/opensearch-project/opensearch-api-specification/pull/546))([#564](https://github.com/opensearch-project/opensearch-api-specification/pull/564)) - Added `h`, `bytes`, `time` parameters to cat API tests ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Added `terminated_early` to `count@200` ([#547](https://github.com/opensearch-project/opensearch-api-specification/pull/547)) - Added request and response schemas for `/_cluster/routing/awareness/{attribute}/weights` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index c532ef260..c4852c848 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -190,7 +190,15 @@ components: minProperties: 1 maxProperties: 1 terms: - $ref: '#/components/schemas/TermsQuery' + type: object + additionalProperties: + anyOf: + - $ref: '#/components/schemas/TermsQuery' + - type: array + items: + type: string + minProperties: 1 + maxProperties: 1 terms_set: description: |- Returns documents that contain a minimum number of exact terms in a provided field. @@ -280,23 +288,6 @@ components: type: number _name: type: string - Terms: - oneOf: - - type: array - items: - type: string - - type: object - properties: - index: - $ref: '_common.yaml#/components/schemas/IndexName' - id: - $ref: '_common.yaml#/components/schemas/Id' - path: - $ref: '_common.yaml#/components/schemas/Field' - routing: - $ref: '_common.yaml#/components/schemas/Routing' - additionalProperties: true - description: Object for fetching terms. BoostingQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -1891,7 +1882,16 @@ components: TermsQuery: anyOf: - $ref: '#/components/schemas/QueryBase' - - $ref: '#/components/schemas/Terms' + - type: object + properties: + index: + $ref: '_common.yaml#/components/schemas/IndexName' + id: + $ref: '_common.yaml#/components/schemas/Id' + path: + $ref: '_common.yaml#/components/schemas/Field' + routing: + $ref: '_common.yaml#/components/schemas/Routing' TermsSetQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -1902,7 +1902,9 @@ components: minimum_should_match_script: $ref: '_common.yaml#/components/schemas/Script' terms: - $ref: '#/components/schemas/Terms' + type: array + items: + type: string required: - terms TextExpansionQuery: From 7b5c70cd0cef4beb04b7436d70a788b4454e0a70 Mon Sep 17 00:00:00 2001 From: "Theo N. Truong" Date: Tue, 10 Sep 2024 07:34:33 -0600 Subject: [PATCH 09/73] Corrected Search Body Sort (#563) --- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 20 ++- TESTING_GUIDE.md | 46 ++--- spec/_global_parameters.yaml | 2 +- spec/schemas/_common.yaml | 5 + tests/default/_core/search/sort_by_field.yaml | 163 ++++++++++++++++++ 6 files changed, 206 insertions(+), 31 deletions(-) create mode 100644 tests/default/_core/search/sort_by_field.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ced75260..a0707cc98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Split test suite ([#472])(https://github.com/opensearch-project/opensearch-api-specification/pull/472) - Changed `WriteResponseBase`'s `_primary_term`, `_seq_no` & `_version` to have `int64` format ([#530](https://github.com/opensearch-project/opensearch-api-specification/pull/530)) - Adjusted indices, shards cat API to test against unassigned indices ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) +- Corrected Sort schema for `/_search`'s request body ([#529](https://github.com/opensearch-project/opensearch-api-specification/issues/529)) - Rename `Bytes` component to `StorageType` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Rename `ByteSize` to `StorageSize` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 43c30d7f3..8314eb633 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -17,15 +17,15 @@ * [Spec Linter](#spec-linter) * [Arguments](#arguments-1) * [Example](#example-1) + * [Spec Tester](#spec-tester) * [Dump Cluster Spec](#dump-cluster-spec) * [Arguments](#arguments-2) * [Example](#example-2) * [Coverage](#coverage) * [Arguments](#arguments-3) * [Example](#example-3) - * [Testing](#testing) - * [Tests](#tests) - * [Lints](#lints) + * [Tools Testing](#tools-testing) + * [Tools Linting](#tools-linting) * [Workflows](#workflows) * [Analyze PR Changes](#analyze-pr-changes) * [Build](#build) @@ -211,6 +211,14 @@ We can take advantage of the default values and simply lint the specification vi npm run lint:spec ``` +### Spec Tester + +```bash +npm run test:spec -- --help +``` + +The spec test framework validates the OpenSearch spec against a running OpenSearch cluster. As you modify the spec, you should add or update the spec test stories in the [./tests](tests) directory. For information on this topic, see [TESTING_GUIDE.md](TESTING_GUIDE.md). + ### [Dump Cluster Spec](tools/src/dump-cluster-spec) ```bash @@ -306,9 +314,7 @@ The output file `build/coverage.json` will now contain data of like below: } ``` -### Testing - -#### Tests +### Tools Testing All tools should have tests added in [tools/tests](tools/tests), tests are implemented using [Jest](https://jestjs.io/). They can be run via: ```bash @@ -322,7 +328,7 @@ npm run jest -- tools/tests/linter/lint.test.ts The test suite contains unit tests and integration tests. Integration tests, such as [these](tools/tests/tester/integ/), require a local instance of OpenSearch and are placed into a folder named `integ`. Unit tests are run in parallel and integration tests are run sequentially using `--runInBand`. You can run unit tests with `npm run test:unit` separately from integration tests with `npm run test:integ`. -#### Lints +### Tools Linting All TypeScript code and YAML files are linted using [ESLint](https://eslint.org/), [typescript-eslint](https://typescript-eslint.io/), and [eslint-plugin-yml](https://ota-meshi.github.io/eslint-plugin-yml/). Linting can be run via: ```bash diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 91b211686..65312657e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,27 +1,27 @@ -- [Spec Testing Guide](#spec-testing-guide) - - [Running Spec Tests](#running-spec-tests) - - [Running Spec Tests Locally](#running-spec-tests-locally) - - [Prerequisites](#prerequisites) - - [OpenSearch Cluster](#opensearch-cluster) - - [Run Tests](#run-tests) - - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) - - [Common Errors](#common-errors) - - [401 Unauthorized](#401-unauthorized) - - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) - - [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) - - [Writing Spec Tests](#writing-spec-tests) - - [Simple Test Story](#simple-test-story) - - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - - [Managing Versions](#managing-versions) - - [Managing Distributions](#managing-distributions) - - [Waiting for Tasks](#waiting-for-tasks) - - [Warnings](#warnings) - - [multiple-paths-detected](#multiple-paths-detected) - - [Suppressing Warnings](#suppressing-warnings) - - [Collecting Test Coverage](#collecting-test-coverage) - - [Coverage Summary](#coverage-summary) - - [Coverage Report](#coverage-report) +* [Spec Testing Guide](#spec-testing-guide) + * [Running Spec Tests](#running-spec-tests) + * [Running Spec Tests Locally](#running-spec-tests-locally) + * [Prerequisites](#prerequisites-) + * [OpenSearch Cluster](#opensearch-cluster-) + * [Run Tests](#run-tests) + * [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) + * [Common Errors](#common-errors) + * [401 Unauthorized](#401-unauthorized) + * [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + * [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) + * [Writing Spec Tests](#writing-spec-tests) + * [Simple Test Story](#simple-test-story) + * [Using Output from Previous Chapters](#using-output-from-previous-chapters) + * [Managing Versions](#managing-versions) + * [Managing Distributions](#managing-distributions) + * [Waiting for Tasks](#waiting-for-tasks) + * [Warnings](#warnings) + * [multiple-paths-detected](#multiple-paths-detected) + * [Suppressing Warnings](#suppressing-warnings) + * [Collecting Test Coverage](#collecting-test-coverage) + * [Coverage Summary](#coverage-summary) + * [Coverage Report](#coverage-report) # Spec Testing Guide diff --git a/spec/_global_parameters.yaml b/spec/_global_parameters.yaml index d3e7f2806..91ddbfe8f 100644 --- a/spec/_global_parameters.yaml +++ b/spec/_global_parameters.yaml @@ -34,7 +34,7 @@ components: filter_path: name: filter_path in: query - description: Comma-separated list of filters used to reduce the response. + description: Used to reduce the response. This parameter takes a comma-separated list of filters. It supports using wildcards to match any field or part of a field’s name. You can also exclude fields with "-". schema: oneOf: - type: string diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 22c70dbfa..a33d64dd8 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -496,7 +496,12 @@ components: SortCombinations: oneOf: - $ref: '#/components/schemas/Field' + - $ref: '#/components/schemas/FieldWithOrder' - $ref: '#/components/schemas/SortOptions' + FieldWithOrder: + type: object + additionalProperties: + $ref: '#/components/schemas/ScoreSort' SortOptions: type: object properties: diff --git a/tests/default/_core/search/sort_by_field.yaml b/tests/default/_core/search/sort_by_field.yaml new file mode 100644 index 000000000..4256929ee --- /dev/null +++ b/tests/default/_core/search/sort_by_field.yaml @@ -0,0 +1,163 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test different ways to sort by field. + +epilogues: + - path: /sorted_movies + method: DELETE + status: [200] + +prologues: + - path: /sorted_movies + method: PUT + request: + payload: + mappings: + properties: + title: + type: keyword + year: + type: integer + + - path: /sorted_movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: The Lion King + year: 1994 + status: [201] + - path: /sorted_movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: The Lion King + year: 2019 + status: [201] + - path: /sorted_movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: Frozen + year: 2013 + status: [201] + +chapters: + - synopsis: QueryString Sort by year. + path: /{index}/_search + parameters: + index: sorted_movies + sort: year + method: GET + response: + status: 200 + payload: + hits: + hits: + - _source: + title: The Lion King + year: 1994 + - _source: + title: Frozen + year: 2013 + - _source: + title: The Lion King + year: 2019 + + - synopsis: QueryString Sort by title:desc. + path: /{index}/_search + parameters: + index: sorted_movies + sort: title:desc + method: GET + response: + status: 200 + payload: + hits: + hits: + - _source: + title: The Lion King + year: 1994 + - _source: + title: The Lion King + year: 2019 + - _source: + title: Frozen + year: 2013 + + - synopsis: QueryString Sort by title:desc,year:desc. + path: /{index}/_search + parameters: + index: sorted_movies + sort: title:desc,year:desc + method: GET + response: + status: 200 + payload: + hits: + hits: + - _source: + title: The Lion King + year: 2019 + - _source: + title: The Lion King + year: 1994 + - _source: + title: Frozen + year: 2013 + + - synopsis: Body Sort by [title, year]. + path: /{index}/_search + parameters: + index: sorted_movies + method: GET + request: + payload: + sort: + - title + - year + response: + status: 200 + payload: + hits: + hits: + - _source: + title: Frozen + year: 2013 + - _source: + title: The Lion King + year: 1994 + - _source: + title: The Lion King + year: 2019 + + - synopsis: Body Sort by year:asc,title:desc. + path: /{index}/_search + parameters: + index: sorted_movies + method: GET + request: + payload: + sort: + - title: {order: asc} + - year: {order: desc} + response: + status: 200 + payload: + hits: + hits: + - _source: + title: Frozen + year: 2013 + - _source: + title: The Lion King + year: 2019 + - _source: + title: The Lion King + year: 1994 + From a51361e842edd98978bffc0fcc821651f3e84e98 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 11 Sep 2024 05:53:09 +1200 Subject: [PATCH 10/73] Fix tasks namespace (#520) --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 126 ++---------- spec/namespaces/tasks.yaml | 2 +- spec/schemas/_common.yaml | 192 ++++++++++++++---- spec/schemas/tasks._common.yaml | 81 +++++--- tests/default/tasks/list.yaml | 25 +++ tools/src/_utils/JsonSchemaValidator.ts | 8 +- .../tests/_utils/JsonSchemaValidator.test.ts | 13 ++ 8 files changed, 265 insertions(+), 183 deletions(-) create mode 100644 tests/default/tasks/list.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a0707cc98..f56751ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561)) - Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) +- Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) ### Security diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index e4eae5c8f..c9d5927be 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2719,42 +2719,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - deleted: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - requests_per_second: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - slice_id: - type: number - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - throttled: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - throttled_until: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - version_conflicts: - type: number + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' delete_by_query_rethrottle@200: content: application/json: @@ -2982,42 +2952,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - created: - type: number - deleted: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - requests_per_second: - type: number - slice_id: - type: number - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/EpochTimeUnitMillis' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/EpochTimeUnitMillis' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - updated: - type: number - version_conflicts: - type: number + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' reindex_rethrottle@200: content: application/json: @@ -3178,42 +3118,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - deleted: - type: number - requests_per_second: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - updated: - type: number - version_conflicts: - type: number - throttled: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - throttled_until: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' update_by_query_rethrottle@200: content: application/json: diff --git a/spec/namespaces/tasks.yaml b/spec/namespaces/tasks.yaml index 013155f8f..a0409b8a6 100644 --- a/spec/namespaces/tasks.yaml +++ b/spec/namespaces/tasks.yaml @@ -90,7 +90,7 @@ components: task: $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskInfo' response: - type: object + $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskResponse' error: $ref: '../schemas/_common.yaml#/components/schemas/ErrorCause' required: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index a33d64dd8..39eca3980 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1420,25 +1420,6 @@ components: type: string enum: - auto - BulkIndexByScrollFailure: - type: object - properties: - cause: - $ref: '#/components/schemas/ErrorCause' - id: - $ref: '#/components/schemas/Id' - index: - $ref: '#/components/schemas/IndexName' - status: - type: number - type: - type: string - required: - - cause - - id - - index - - status - - type Retries: type: object properties: @@ -1948,23 +1929,32 @@ components: items: $ref: '#/components/schemas/NodeRole' NodeRole: - type: string - enum: - - client - - cluster_manager - - coordinating_only - - data - - data_cold - - data_content - - data_frozen - - data_hot - - data_warm - - ingest - - master - - ml - - remote_cluster_client - - transform - - voting_only + oneOf: + - type: string + enum: + - client + - coordinating_only + - data + - data_cold + - data_content + - data_frozen + - data_hot + - data_warm + - ingest + - ml + - remote_cluster_client + - transform + - voting_only + - type: string + enum: + - master + deprecated: true + x-version-deprecated: '2.0' + x-deprecation-message: To promote inclusive language, use 'cluster_manager' instead. + - type: string + enum: + - cluster_manager + x-version-added: '2.0' HttpHeaders: type: object additionalProperties: @@ -1995,11 +1985,7 @@ components: transport_address: $ref: '#/components/schemas/TransportAddress' required: - - attributes - - host - - ip - name - - transport_address RankContainer: type: object properties: @@ -2116,3 +2102,129 @@ components: - expand - fetch - query + BulkByScrollTaskStatus: + type: object + properties: + slice_id: + type: integer + format: int32 + total: + description: The number of documents that were successfully processed. + type: integer + format: int64 + updated: + description: The number of documents that were successfully updated, for example, a document with same ID already existed prior to reindex updating it. + type: integer + format: int64 + created: + description: The number of documents that were successfully created. + type: integer + format: int64 + deleted: + description: The number of documents that were successfully deleted. + type: integer + format: int64 + batches: + description: The number of scroll responses pulled back by the reindex. + type: integer + format: int32 + version_conflicts: + description: The number of version conflicts that reindex hits. + type: integer + format: int64 + noops: + description: The number of documents that were ignored. + type: integer + format: int64 + retries: + $ref: '#/components/schemas/Retries' + throttled_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' + throttled: + $ref: '#/components/schemas/Duration' + requests_per_second: + description: The number of requests per second effectively executed during the reindex. + type: number + format: float + canceled: + type: string + throttled_until_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' + throttled_until: + $ref: '#/components/schemas/Duration' + slices: + type: array + items: + $ref: '#/components/schemas/BulkByScrollTaskStatusOrException' + required: + - batches + - deleted + - noops + - requests_per_second + - retries + - throttled_millis + - throttled_until_millis + - total + - version_conflicts + BulkByScrollTaskStatusOrException: + oneOf: + - title: status + $ref: '#/components/schemas/BulkByScrollTaskStatus' + - title: exception + $ref: '#/components/schemas/ErrorCause' + BulkByScrollResponseBase: + allOf: + - $ref: '#/components/schemas/BulkByScrollTaskStatus' + - type: object + properties: + took: + type: integer + format: int64 + timed_out: + type: boolean + failures: + type: array + items: + $ref: '#/components/schemas/BulkByScrollFailure' + required: + - failures + - timed_out + - took + BulkByScrollFailure: + anyOf: + - $ref: '#/components/schemas/BulkItemResponseFailure' + - $ref: '#/components/schemas/ScrollableHitSourceSearchFailure' + BulkItemResponseFailure: + type: object + properties: + cause: + $ref: '#/components/schemas/ErrorCause' + id: + $ref: '#/components/schemas/Id' + index: + $ref: '#/components/schemas/IndexName' + status: + type: integer + format: int32 + required: + - cause + - index + - status + ScrollableHitSourceSearchFailure: + type: object + properties: + index: + $ref: '#/components/schemas/IndexName' + shard: + type: integer + format: int32 + node: + type: string + status: + type: integer + format: int32 + reason: + $ref: '#/components/schemas/ErrorCause' + required: + - reason + - status diff --git a/spec/schemas/tasks._common.yaml b/spec/schemas/tasks._common.yaml index 8e5399cb4..c5afaccda 100644 --- a/spec/schemas/tasks._common.yaml +++ b/spec/schemas/tasks._common.yaml @@ -21,34 +21,20 @@ components: description: Task information grouped by node, if `group_by` was set to `node` (the default). type: object additionalProperties: - $ref: '#/components/schemas/NodeTasks' + $ref: '#/components/schemas/TaskExecutingNode' tasks: $ref: '#/components/schemas/TaskInfos' - NodeTasks: - type: object - properties: - name: - $ref: '_common.yaml#/components/schemas/NodeId' - transport_address: - $ref: '_common.yaml#/components/schemas/TransportAddress' - host: - $ref: '_common.yaml#/components/schemas/Host' - ip: - $ref: '_common.yaml#/components/schemas/Ip' - roles: - type: array - items: - type: string - attributes: - type: object - additionalProperties: - type: string - tasks: - type: object - additionalProperties: - $ref: '#/components/schemas/TaskInfo' - required: - - tasks + TaskExecutingNode: + allOf: + - $ref: '_common.yaml#/components/schemas/BaseNode' + - type: object + properties: + tasks: + type: object + additionalProperties: + $ref: '#/components/schemas/TaskInfo' + required: + - tasks TaskInfo: type: object properties: @@ -76,8 +62,7 @@ components: start_time_in_millis: $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' status: - description: Task status information can vary wildly from task to task. - type: object + $ref: '#/components/schemas/Status' type: type: string parent_task_id: @@ -93,13 +78,15 @@ components: - type TaskInfos: oneOf: - - type: array + - title: grouped_by_none + type: array items: $ref: '#/components/schemas/TaskInfo' - - type: object + - title: grouped_by_parents + type: object additionalProperties: - $ref: '#/components/schemas/ParentTaskInfo' - ParentTaskInfo: + $ref: '#/components/schemas/TaskGroup' + TaskGroup: allOf: - $ref: '#/components/schemas/TaskInfo' - type: object @@ -107,10 +94,38 @@ components: children: type: array items: - $ref: '#/components/schemas/TaskInfo' + $ref: '#/components/schemas/TaskGroup' GroupBy: type: string enum: - nodes - none - parents + Status: + description: Task status information can vary wildly from task to task. + anyOf: + - $ref: '#/components/schemas/ReplicationTaskStatus' + - $ref: '_common.yaml#/components/schemas/BulkByScrollTaskStatus' + - $ref: '#/components/schemas/PersistentTaskStatus' + - $ref: '#/components/schemas/RawTaskStatus' + ReplicationTaskStatus: + type: object + properties: + phase: + type: string + required: + - phase + PersistentTaskStatus: + type: object + properties: + state: + type: string + required: + - state + RawTaskStatus: + type: object + additionalProperties: + title: metadata + TaskResponse: + anyOf: + - $ref: '_common.yaml#/components/schemas/BulkByScrollResponseBase' diff --git a/tests/default/tasks/list.yaml b/tests/default/tasks/list.yaml new file mode 100644 index 000000000..a687e170c --- /dev/null +++ b/tests/default/tasks/list.yaml @@ -0,0 +1,25 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test tasks list endpoint. +chapters: + - synopsis: List tasks grouped by node. + path: /_tasks + method: GET + parameters: + group_by: nodes + response: + status: 200 + - synopsis: List tasks grouped by parent. + path: /_tasks + method: GET + parameters: + group_by: parents + response: + status: 200 + - synopsis: List tasks grouped by none. + path: /_tasks + method: GET + parameters: + group_by: none + response: + status: 200 \ No newline at end of file diff --git a/tools/src/_utils/JsonSchemaValidator.ts b/tools/src/_utils/JsonSchemaValidator.ts index ff33dbc1f..850468363 100644 --- a/tools/src/_utils/JsonSchemaValidator.ts +++ b/tools/src/_utils/JsonSchemaValidator.ts @@ -38,7 +38,13 @@ export default class JsonSchemaValidator { addFormats(this.ajv); if (options.ajv_errors_opts != null) ajv_errors(this.ajv, options.ajv_errors_opts) for (const keyword of options.additional_keywords ?? []) this.ajv.addKeyword(keyword) - Object.entries(options.reference_schemas ?? {}).forEach(([key, schema]) => this.ajv.addSchema(schema, key)) + Object.entries(options.reference_schemas ?? {}).forEach(([key, schema]) => { + try { + this.ajv.addSchema(schema, key); + } catch (e) { + throw new Error(`Failed to add schema ${key} (${JSON.stringify(schema)}):\n\t${e instanceof Error ? e.message : e as string}`) + } + }) this.errors_parser = new AjvErrorsParser(this.ajv, options.errors_text_opts) if (default_schema) this._validate = this.ajv.compile(default_schema) } diff --git a/tools/tests/_utils/JsonSchemaValidator.test.ts b/tools/tests/_utils/JsonSchemaValidator.test.ts index 474dc1206..83b7c29a8 100644 --- a/tools/tests/_utils/JsonSchemaValidator.test.ts +++ b/tools/tests/_utils/JsonSchemaValidator.test.ts @@ -83,4 +83,17 @@ describe('JsonSchemaValidator', () => { expect(validator.validate_schema(invalid_schema)).toEqual('data/required must be array'); }); + + test('constructing with invalid reference schema throws descriptive error', () => { + const invalid_options = { + reference_schemas: { + '#/components/schemas/Invalid': { + type: 'string', + required: true, // required must be an array + } + } + } + expect(() => new JsonSchemaValidator(schema, invalid_options)) + .toThrow('Failed to add schema #/components/schemas/Invalid ({"type":"string","required":true}):\n\tschema is invalid: data/required must be array'); + }); }); \ No newline at end of file From a499dbf972414c352c10f20bbffa0e51c1e50e55 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 10 Sep 2024 19:28:37 -0400 Subject: [PATCH 11/73] Show relative path instead of a full one. (#566) Signed-off-by: dblock --- tools/src/tester/ResultLogger.ts | 3 ++- tools/tests/tester/test.test.ts | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts index 109b05a7c..a5357882b 100644 --- a/tools/src/tester/ResultLogger.ts +++ b/tools/src/tester/ResultLogger.ts @@ -12,6 +12,7 @@ import { overall_result } from './helpers' import * as ansi from './Ansi' import TestResults from './TestResults' import _ from 'lodash' +import * as path from 'path' export interface ResultLogger { log: (evaluation: StoryEvaluation) => void @@ -71,7 +72,7 @@ export class ConsoleResultLogger implements ResultLogger { } #log_story ({ result, full_path, display_path, message, warnings }: StoryEvaluation): void { - this.#log_evaluation({ result, message: message ?? full_path }, ansi.cyan(ansi.b(display_path))) + this.#log_evaluation({ result, message: message ?? path.relative('.', full_path) }, ansi.cyan(ansi.b(display_path))) this.#log_warnings(warnings) } diff --git a/tools/tests/tester/test.test.ts b/tools/tests/tester/test.test.ts index e6677185d..fd9949fc8 100644 --- a/tools/tests/tester/test.test.ts +++ b/tools/tests/tester/test.test.ts @@ -9,7 +9,6 @@ import { spawnSync } from 'child_process' import * as ansi from 'tester/Ansi' -import * as path from 'path' import { type Chapter, type ChapterRequest, type Output, type Request, Story } from 'tester/types/story.types' import { ChapterEvaluation, Result, StoryEvaluation } from 'tester/types/eval.types' import StoryEvaluator from 'tester/StoryEvaluator' @@ -158,11 +157,10 @@ test.todo('--tab-width') test('--dry-run', () => { const test_yaml = 'tools/tests/tester/fixtures/empty_with_all_the_parts.yaml' const s = spec(['--dry-run', '--tests', test_yaml]).stdout - const full_path = path.join(__dirname, '../../../' + test_yaml) expect(s).not.toContain(`${ansi.yellow('SKIPPED')} CHAPTERS`) expect(s).not.toContain(`${ansi.yellow('SKIPPED')} EPILOGUES`) expect(s).not.toContain(`${ansi.yellow('SKIPPED')} PROLOGUES`) - expect(s).toContain(`${ansi.yellow('SKIPPED')} ${ansi.cyan(ansi.b('empty_with_all_the_parts.yaml'))} ${ansi.gray('(' + full_path + ')')}`) + expect(s).toContain(`${ansi.yellow('SKIPPED')} ${ansi.cyan(ansi.b('empty_with_all_the_parts.yaml'))} ${ansi.gray('(' + test_yaml + ')')}`) }) test('--dry-run --verbose', () => { From 9d59d92832420a60b9599c96f7a4b44d1193137d Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 11 Sep 2024 15:02:04 -0400 Subject: [PATCH 12/73] Added tests for _field_caps. (#567) --- tests/default/_core/field_caps.yaml | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/default/_core/field_caps.yaml diff --git a/tests/default/_core/field_caps.yaml b/tests/default/_core/field_caps.yaml new file mode 100644 index 000000000..342247e64 --- /dev/null +++ b/tests/default/_core/field_caps.yaml @@ -0,0 +1,53 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test field capabilities API. +prologues: + - path: /movies + method: PUT +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get information about the capabilities of all fields. + path: /_field_caps + method: GET + parameters: + fields: '*' + allow_no_indices: false + expand_wildcards: all + ignore_unavailable: true + include_unmapped: true + response: + status: 200 + - synopsis: Get information about the capabilities of all fields for a given index. + path: /{index}/_field_caps + method: GET + parameters: + fields: type + index: movies + response: + status: 200 + - synopsis: Get information about the capabilities of specific fields. + path: /_field_caps + method: POST + parameters: + fields: type + request: + payload: + index_filter: + match_all: {} + response: + status: 200 + - synopsis: Get information about the capabilities of all fields for a given index. + path: /{index}/_field_caps + method: POST + parameters: + fields: type + index: movies + request: + payload: + index_filter: + match_all: {} + response: + status: 200 From d05ba73a29add46933e74db19cb1de0103fefc1c Mon Sep 17 00:00:00 2001 From: Junwei Dai <59641585+junweid62@users.noreply.github.com> Date: Wed, 11 Sep 2024 13:44:11 -0700 Subject: [PATCH 13/73] Complete API spec implementation for flow_framework (#565) * Add some 4xx error test Signed-off-by: Junwei Dai * 1. Adding API spec for GET /workflow/_steps 2. Adding Test for this API Signed-off-by: Junwei Dai * 1. Adding API spec for GET /workflow/_status 2. Adding Test for this API Signed-off-by: Junwei Dai * 1. Adding API spec for POST /workflow/_deprovision 2. Adding Test for this API Signed-off-by: Junwei Dai * 1. Adding API spec for POST /workflow/_provision 2. Adding Test for this API Signed-off-by: Junwei Dai * 1. Adding API spec for POST /workflow/_search 2. Adding Test for this API Signed-off-by: Junwei Dai * 1. Adding API spec for POST/GET /workflow/state/_search 2. Adding Test for this API Signed-off-by: Junwei Dai * add change log Signed-off-by: Junwei Dai * Update tests/default/flow_framework/workflow.yaml Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> * change searchStateResponse spec. Signed-off-by: Junwei Dai * update workflow.yaml add new test file seperate test senario Signed-off-by: Junwei Dai --------- Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> Co-authored-by: Junwei Dai Co-authored-by: Daniel (dB.) Doubrovkine --- .cspell | 3 +- CHANGELOG.md | 1 + spec/namespaces/flow_framework.yaml | 374 +++++++++++++++++- spec/schemas/flow_framework.common.yaml | 246 +++++++++++- spec/schemas/flow_framework.errors.yaml | 82 +++- tests/default/flow_framework/deprovision.yaml | 63 +++ tests/default/flow_framework/provision.yaml | 55 +++ tests/default/flow_framework/search.yaml | 40 ++ .../default/flow_framework/search_state.yaml | 31 ++ tests/default/flow_framework/status.yaml | 37 ++ tests/default/flow_framework/steps.yaml | 33 ++ tests/default/flow_framework/workflow.yaml | 99 ++++- 12 files changed, 1033 insertions(+), 31 deletions(-) create mode 100644 tests/default/flow_framework/deprovision.yaml create mode 100644 tests/default/flow_framework/provision.yaml create mode 100644 tests/default/flow_framework/search.yaml create mode 100644 tests/default/flow_framework/search_state.yaml create mode 100644 tests/default/flow_framework/status.yaml create mode 100644 tests/default/flow_framework/steps.yaml diff --git a/.cspell b/.cspell index 518d11f61..df8953cd6 100644 --- a/.cspell +++ b/.cspell @@ -192,4 +192,5 @@ vectory whoamiprotected wordnet Yrtsd -reprovision \ No newline at end of file +reprovision +deprovision \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index f56751ee4..002280bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) +- Added `/_plugins/_flow_framework/` ,`_search`,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ### Changed diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index eff3c923a..e3a3e005c 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -15,6 +15,7 @@ paths: parameters: - $ref: '#/components/parameters/flow_framework.create::query.provision' - $ref: '#/components/parameters/flow_framework.create::query.reprovision' + - $ref: '#/components/parameters/flow_framework.create::query.update_fields' - $ref: '#/components/parameters/flow_framework.create::query.use_case' - $ref: '#/components/parameters/flow_framework.create::query.validation' requestBody: @@ -50,7 +51,11 @@ paths: url: https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/ parameters: - $ref: '#/components/parameters/flow_framework.update::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.update::query.provision' + - $ref: '#/components/parameters/flow_framework.update::query.reprovision' - $ref: '#/components/parameters/flow_framework.update::query.update_fields' + - $ref: '#/components/parameters/flow_framework.update::query.use_case' + - $ref: '#/components/parameters/flow_framework.update::query.validation' requestBody: $ref: '#/components/requestBodies/flow_framework.update' responses: @@ -59,6 +64,8 @@ paths: description: It returns the workflow_id '404': $ref: '#/components/responses/flow_framework.update@404' + '400': + $ref: '#/components/responses/flow_framework.update@400' delete: operationId: flow_framework.delete.0 x-operation-group: flow_framework.delete @@ -76,8 +83,158 @@ paths: $ref: '#/components/responses/flow_framework.delete@400' '403': $ref: '#/components/responses/flow_framework.delete@403' + /_plugins/_flow_framework/workflow/_steps: + get: + operationId: flow_framework.get_steps.0 + x-operation-group: flow_framework.get_steps + x-version-added: '2.12' + description: Get a list of workflow steps. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/get-workflow-steps/ + parameters: + - $ref: '#/components/parameters/flow_framework.get_steps::query.workflow_step' + responses: + '200': + $ref: '#/components/responses/flow_framework.get_steps@200' + '403': + $ref: '#/components/responses/flow_framework.get_steps@403' + '400': + $ref: '#/components/responses/flow_framework.get_steps@400' + /_plugins/_flow_framework/workflow/{workflow_id}/_status: + get: + operationId: flow_framework.get_status.0 + x-operation-group: flow_framework.get_status + x-version-added: '2.12' + description: Get the provisioning deployment status until it is complete. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/get-workflow-status/ + parameters: + - $ref: '#/components/parameters/flow_framework.get_status::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.get_status::query.all' + responses: + '200': + $ref: '#/components/responses/flow_framework.get_status@200' + '403': + $ref: '#/components/responses/flow_framework.get_status@403' + '404': + $ref: '#/components/responses/flow_framework.get_status@404' + /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision: + post: + operationId: flow_framework.deprovision.0 + x-operation-group: flow_framework.deprovision + x-version-added: '2.12' + description: Deprovision workflow's resources when you no longer need it. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/deprovision-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.deprovision::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.deprovision::query.allow_delete' + responses: + '200': + $ref: '#/components/responses/flow_framework.deprovision@200' + '202': + $ref: '#/components/responses/flow_framework.deprovision@202' + '403': + $ref: '#/components/responses/flow_framework.deprovision@403' '404': - $ref: '#/components/responses/flow_framework.delete@404' + $ref: '#/components/responses/flow_framework.deprovision@404' + /_plugins/_flow_framework/workflow/{workflow_id}/_provision: + post: + operationId: flow_framework.provision.0 + x-operation-group: flow_framework.provision + x-version-added: '2.12' + description: Provisioning a workflow. This API is also executed when the Create or Update Workflow API is called with the provision parameter set to true. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.provision::path.workflow_id' + requestBody: + description: |- + Pass dynamic substitution expressions as key-value pairs, either as query parameters or in the request body. + For example, if you specified a credential field in the template as `openai_key: '$'`, + you can pass `openai_key` as a query parameter or in the request body. + $ref: '#/components/requestBodies/flow_framework.provision' + responses: + '200': + $ref: '#/components/responses/flow_framework.provision@200' + '403': + $ref: '#/components/responses/flow_framework.provision@403' + '400': + $ref: '#/components/responses/flow_framework.provision@400' + /_plugins/_flow_framework/workflow/_search: + get: + operationId: flow_framework.search.1 + x-operation-group: flow_framework.search + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search' + responses: + '200': + $ref: '#/components/responses/flow_framework.search@200' + '403': + $ref: '#/components/responses/flow_framework.search@403' + '408': + $ref: '#/components/responses/flow_framework.search@408' + '400': + $ref: '#/components/responses/flow_framework.search@400' + post: + operationId: flow_framework.search.0 + x-operation-group: flow_framework.search + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search' + responses: + '200': + $ref: '#/components/responses/flow_framework.search@200' + '403': + $ref: '#/components/responses/flow_framework.search@403' + '408': + $ref: '#/components/responses/flow_framework.search@408' + '400': + $ref: '#/components/responses/flow_framework.search@400' + /_plugins/_flow_framework/workflow/state/_search: + get: + operationId: flow_framework.search_state.1 + x-operation-group: flow_framework.search_state + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/search-workflow-state/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search_state' + responses: + '200': + $ref: '#/components/responses/flow_framework.search_state@200' + '403': + $ref: '#/components/responses/flow_framework.search_state@403' + '408': + $ref: '#/components/responses/flow_framework.search_state@408' + '400': + $ref: '#/components/responses/flow_framework.search_state@400' + post: + operationId: flow_framework.search_state.0 + x-operation-group: flow_framework.search_state + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/search-workflow-state/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search_state' + responses: + '200': + $ref: '#/components/responses/flow_framework.search_state@200' + '403': + $ref: '#/components/responses/flow_framework.search_state@403' + '408': + $ref: '#/components/responses/flow_framework.search_state@408' + '400': + $ref: '#/components/responses/flow_framework.search_state@400' components: parameters: flow_framework.get::path.workflow_id: @@ -85,33 +242,64 @@ components: in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.get_status::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.deprovision::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.provision::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.update::path.workflow_id: name: workflow_id in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.delete::path.workflow_id: name: workflow_id in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.create::query.provision: in: query name: provision schema: - description: Provision the workflow as part of the request - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Provision' + flow_framework.update::query.provision: + in: query + name: provision + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Provision' flow_framework.create::query.reprovision: x-version-added: '2.17' in: query name: reprovision schema: - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' + flow_framework.deprovision::query.allow_delete: + in: query + name: allow_delete + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/AllowDelete' + flow_framework.update::query.reprovision: + x-version-added: '2.17' + in: query + name: reprovision + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' flow_framework.create::query.use_case: description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. externalDocs: @@ -119,21 +307,41 @@ components: in: query name: use_case schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UseCase' + flow_framework.get_status::query.all: + description: The all parameter specifies whether the response should return all fields. + in: query + name: all + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/All' + flow_framework.update::query.use_case: + description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/workflow-templates/#supported-workflow-templates + in: query + name: use_case + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UseCase' flow_framework.create::query.validation: in: query name: validation schema: - description: Validate the workflow. Valid values are all (validate the template) and none (do not validate the template). Default is all. - type: string - default: all + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Validation' + flow_framework.update::query.validation: + in: query + name: validation + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Validation' flow_framework.update::query.update_fields: in: query name: update_fields schema: - description: Update only the fields included in the request body. - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UpdateFields' + flow_framework.create::query.update_fields: + in: query + name: update_fields + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UpdateFields' flow_framework.delete::query.clear_status: in: query name: clear_status @@ -141,6 +349,13 @@ components: description: Delete the workflow state (without deprovisioning resources) after deleting the template. OpenSearch deletes the workflow state only if the provisioning status is not IN_PROGRESS. Default is false. type: boolean default: false + flow_framework.get_steps::query.workflow_step: + in: query + name: workflow_step + required: false + schema: + description: The name of the step to retrieve. + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowStepName' requestBodies: flow_framework.create: content: @@ -152,6 +367,24 @@ components: application/json: schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkUpdate' + flow_framework.provision: + required: false + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UserProvidedSubstitutionExpressions' + flow_framework.search: + required: true + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/SearchWorkflowRequest' + flow_framework.search_state: + required: true + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/SearchWorkflowRequest' responses: flow_framework.create@201: content: @@ -173,7 +406,18 @@ components: content: application/json: oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + flow_framework.update@400: + description: Bad Request - Multiple possible reasons + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' @@ -213,13 +457,101 @@ components: content: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' - flow_framework.delete@404: + flow_framework.get@404: description: Template Not Found Error content: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' - flow_framework.get@404: - description: Template Not Found Error + flow_framework.get_steps@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowStep' + flow_framework.get_steps@403: + description: Flow Framework API Disabled Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.get_steps@400: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowStepsRetrieveError' + flow_framework.get_status@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.get_status@404: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.get_status@200: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusFullResponse' + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusDefaultResponse' + flow_framework.deprovision@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.provision@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.deprovision@202: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningError' + flow_framework.deprovision@403: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningForbiddenError' + flow_framework.deprovision@404: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.provision@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.provision@400: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestBodyParsingFailedError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidRequestBodyFieldError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DuplicateKeyError' + flow_framework.search@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + flow_framework.search@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.search@408: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + flow_framework.search@400: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + flow_framework.search_state@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchStateResponse' + flow_framework.search_state@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.search_state@408: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + flow_framework.search_state@400: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' \ No newline at end of file + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' \ No newline at end of file diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 6ff3cda19..a8d7d7c00 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -6,6 +6,34 @@ info: paths: { } components: schemas: + AllowDelete: + description: Required when deleting resources involves a high potential for data loss. + type: string + WorkflowID: + type: string + WorkflowStepName: + type: string + Provision: + description: Provision the workflow as part of the request. + type: boolean + default: false + Reprovision: + type: boolean + default: false + UseCase: + type: string + Validation: + description: Validate the workflow. Valid values are all (validate the template) and none (do not validate the template). Default is all. + type: string + default: all + UpdateFields: + description: Update only the fields included in the request body. + type: boolean + default: false + All: + description: The all parameter specifies whether the response should return all fields. + type: boolean + default: false FlowFrameworkCreate: type: object properties: @@ -105,6 +133,82 @@ components: failed: type: integer description: The number of shards where the deletion failed. + query: + type: object + description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`. + properties: + match: + type: object + match_all: + type: object + total: + type: object + properties: + value: + type: integer + description: Total number of matching documents. + relation: + type: string + description: The relation of the total hits. + itemsObject: + type: object + properties: + _index: + type: string + description: The index name where the document resides. + _id: + type: string + description: The document ID. + _version: + type: integer + description: The version of the document. + _seq_no: + type: integer + description: The sequence number of the document. + _primary_term: + type: integer + description: The primary term of the document. + _score: + type: number + format: float + description: The score of the search hit. + _source: + $ref: '#/components/schemas/FlowFrameworkGetResponse' + StateItems: + type: object + properties: + _index: + type: string + description: The index name where the document resides. + _id: + type: string + description: The document ID. + _version: + type: integer + description: The version of the document. + _seq_no: + type: integer + description: The sequence number of the document. + _primary_term: + type: integer + description: The primary term of the document. + _score: + type: number + format: float + description: The score of the search hit. + _source: + $ref: '#/components/schemas/SearchStateResponse' + SearchStateResponse: + type: object + properties: + workflow_id: + type: string + provisioning_progress: + type: string + state: + type: string + user: + $ref: '#/components/schemas/user' user: type: object properties: @@ -124,4 +228,144 @@ components: type: string user_requested_tenant: type: string - nullable: true \ No newline at end of file + nullable: true + hits: + type: object + properties: + total: + $ref: '#/components/schemas/total' + max_score: + type: number + format: float + description: The maximum score of the search hits. + hits: + type: array + items: + $ref: '#/components/schemas/itemsObject' + StateHits: + type: object + properties: + total: + $ref: '#/components/schemas/total' + max_score: + type: number + format: float + description: The maximum score of the search hits. + hits: + type: array + items: + $ref: '#/components/schemas/StateItems' + WorkflowStep: + type: object + properties: + inputs: + type: array + items: + type: string + description: The list of required inputs for the workflow step + outputs: + type: array + items: + type: string + description: The list of outputs generated by the workflow step + required_plugins: + type: array + items: + type: string + description: The list of plugins required for the workflow step + WorkFlowStatusDefaultResponse: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow + error: + type: string + description: Any error state associated with the workflow + nullable: true + state: + type: string + description: The current state of the workflow + resources_created: + type: array + items: + type: string + description: A list of resources created by the workflow + WorkFlowStatusFullResponse: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow + error: + type: string + description: Any error state associated with the workflow + nullable: true + state: + type: string + description: The current state of the workflow + enum: + - COMPLETED + - FAILED + - NOT_STARTED + - PROVISIONING + resources_created: + type: array + items: + type: string + description: A list of resources created by the workflow + provisioning_progress: + type: string + description: The progress of the provisioning process + nullable: true + provision_start_time: + type: string + format: date-time + description: The start time of the provisioning process + provision_end_time: + type: string + format: date-time + description: The end time of the provisioning process + user: + $ref: '#/components/schemas/user' + user_outputs: + type: array + items: + type: string + description: Outputs generated by the user + UserProvidedSubstitutionExpressions: + type: object + additionalProperties: + type: string + description: The value for the user-provided key. Keys represent substitution expressions. + SearchWorkflowRequest: + type: object + properties: + query: + $ref: '#/components/schemas/query' + WorkflowSearchResponse: + type: object + properties: + took: + type: integer + description: Time in milliseconds that the request took to complete. + timed_out: + type: boolean + description: Indicates if the request timed out. + _shards: + $ref: '#/components/schemas/shards' + hits: + $ref: '#/components/schemas/hits' + WorkflowSearchStateResponse: + type: object + properties: + took: + type: integer + description: Time in milliseconds that the request took to complete. + timed_out: + type: boolean + description: Indicates if the request timed out. + _shards: + $ref: '#/components/schemas/shards' + hits: + $ref: '#/components/schemas/StateHits' \ No newline at end of file diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 0224b1cba..6f46f2435 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -16,7 +16,7 @@ components: example: This API is disabled. To enable it, set [flow_framework.enabled] to true. status: type: integer - BadRequestError: + InvalidParameterError: content: application/json: type: object @@ -66,6 +66,35 @@ components: example: Workflow ID can not be null status: type: integer + DuplicateKeyError: + type: object + properties: + error: + type: string + description: Error message when a duplicate key is found in the request. + status: + type: integer + description: HTTP status code for the error. + example: 400 + InvalidRequestBodyFieldError: + type: object + properties: + error: + type: string + description: Error message when a request body field is not a string. + status: + type: integer + description: HTTP status code for the error. + RequestBodyParsingFailedError: + type: object + properties: + error: + type: string + description: Error message when request body parsing fails. + status: + type: integer + description: HTTP status code for the error. + example: 400 WorkflowSaveError: content: application/json: @@ -95,4 +124,53 @@ components: type: string example: Failed to retrieve template (12345) from global context. code: - type: integer \ No newline at end of file + type: integer + WorkflowStepsRetrieveError: + content: + application/json: + type: object + properties: + message: + type: string + example: Failed to retrieve workflow step json. + code: + type: integer + DeprovisioningError: + type: object + properties: + error: + type: string + description: Describes the deprovisioning error and identifies resources that were not deprovisioned + example: Failed to deprovision some resources [connector_id Lw7PX4wBfVtHp98y06wV]. + required: + - error + DeprovisioningForbiddenError: + type: object + properties: + error: + type: string + description: Describes the resources that require the allow_delete parameter for deprovisioning + example: These resources require the allow_delete parameter to deprovision [index_name my-index]. + required: + - error + RequestTimeoutError: + type: object + properties: + error: + type: string + description: Error message when the request times out. + status: + type: integer + description: HTTP status code for the error. + example: 408 + BadRequestError: + type: object + properties: + error: + type: string + description: Error message when the request body or parameters are invalid. + example: Invalid request body or query parameters. + status: + type: integer + description: HTTP status code for the error. + example: 400 \ No newline at end of file diff --git a/tests/default/flow_framework/deprovision.yaml b/tests/default/flow_framework/deprovision.yaml new file mode 100644 index 000000000..43babeeb3 --- /dev/null +++ b/tests/default/flow_framework/deprovision.yaml @@ -0,0 +1,63 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + provision: true + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Check the provision status before calling the deprovision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: PROVISIONING + - synopsis: Deprovision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Check the provision status after calling the deprovision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: NOT_STARTED + - synopsis: Deprovision workflow using an invalid ID. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: invalid + response: + status: 404 + + diff --git a/tests/default/flow_framework/provision.yaml b/tests/default/flow_framework/provision.yaml new file mode 100644 index 000000000..2d52e83e1 --- /dev/null +++ b/tests/default/flow_framework/provision.yaml @@ -0,0 +1,55 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Check the status before calling provision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: NOT_STARTED + - synopsis: Provision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_provision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + request: + payload: + openai_key: '1234556' + response: + status: 200 + - synopsis: Check the provision status after calling provision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: PROVISIONING \ No newline at end of file diff --git a/tests/default/flow_framework/search.yaml b/tests/default/flow_framework/search.yaml new file mode 100644 index 000000000..3e3353079 --- /dev/null +++ b/tests/default/flow_framework/search.yaml @@ -0,0 +1,40 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Search workflow match the name. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match: + name: test + response: + status: 200 + - synopsis: Search all workflow. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match_all: {} + response: + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/search_state.yaml b/tests/default/flow_framework/search_state.yaml new file mode 100644 index 000000000..7dc59ba04 --- /dev/null +++ b/tests/default/flow_framework/search_state.yaml @@ -0,0 +1,31 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Search workflow state. + path: /_plugins/_flow_framework/workflow/state/_search + method: POST + request: + payload: + query: + match: + state: NOT_STARTED + response: + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/status.yaml b/tests/default/flow_framework/status.yaml new file mode 100644 index 000000000..fd9ea2300 --- /dev/null +++ b/tests/default/flow_framework/status.yaml @@ -0,0 +1,37 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Get workflow status. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + state: NOT_STARTED + - synopsis: Get workflow status using an invalid ID. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: invalid + response: + status: 404 \ No newline at end of file diff --git a/tests/default/flow_framework/steps.yaml b/tests/default/flow_framework/steps.yaml new file mode 100644 index 000000000..9e4bc1704 --- /dev/null +++ b/tests/default/flow_framework/steps.yaml @@ -0,0 +1,33 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Get workflow steps. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + response: + status: 200 + - synopsis: Get workflow steps with invalid query parameter. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + parameters: + workflow_step: invalid_query + response: + status: 400 \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 53da81c66..d136c8047 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -19,8 +19,79 @@ chapters: status: 201 output: test_workflow_id: payload.workflow_id + - synopsis: Create workflow With Provision and updateFields set true. + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + provision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With provision and update_fields set. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + provision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With Reprovision and updateFields set true. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow With Reprovision set true. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + reprovision: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow With Reprovision set true and UseCase Not null. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With Reprovision set true and UseCase Not null. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 - synopsis: Get workflow. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: GET parameters: @@ -28,7 +99,6 @@ chapters: response: status: 200 - synopsis: Update workflow. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: @@ -39,21 +109,38 @@ chapters: response: status: 201 - synopsis: Update workflow with a non-existent workflow ID. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: - workflow_id: test + workflow_id: invalid request: payload: name: test_update_work_flow response: status: 404 + - synopsis: Search workflow state. + path: /_plugins/_flow_framework/workflow/state/_search + method: POST + request: + payload: + query: + match: + state: test + response: + status: 200 - synopsis: Delete workflow. - id: delete_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE parameters: workflow_id: ${create_flow_framework.test_workflow_id} response: - status: 200 \ No newline at end of file + status: 200 + - synopsis: Delete workflow using an invalid ID. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + parameters: + workflow_id: invalid + response: + status: 200 + payload: + result: not_found \ No newline at end of file From 508c2106ba8fc7790a6b3eaa0762c84878bae76a Mon Sep 17 00:00:00 2001 From: Jakob Date: Thu, 12 Sep 2024 00:06:42 +0200 Subject: [PATCH 14/73] Rename Bytes Components (#569) * spec: Rename StorageType to ByteUnit Signed-off-by: Jakob Hahn * spec: rename Bytes to ByteCount Signed-off-by: Jakob Hahn * spec: rename StorageSize to HumanReadableByteCount Signed-off-by: Jakob Hahn --------- Signed-off-by: Jakob Hahn --- spec/namespaces/cat.yaml | 20 +- spec/namespaces/indices.yaml | 4 +- spec/schemas/_common.yaml | 139 +++++++------- spec/schemas/cat.segments.yaml | 2 +- spec/schemas/cluster.allocation_explain.yaml | 18 +- spec/schemas/cluster.stats.yaml | 56 +++--- spec/schemas/indices._common.yaml | 12 +- spec/schemas/indices.data_streams_stats.yaml | 4 +- spec/schemas/indices.recovery.yaml | 16 +- spec/schemas/indices.rollover.yaml | 16 +- spec/schemas/indices.segments.yaml | 8 +- spec/schemas/indices.stats.yaml | 10 +- spec/schemas/nodes._common.yaml | 184 +++++++++---------- spec/schemas/nodes.info.yaml | 34 ++-- spec/schemas/snapshot._common.yaml | 8 +- 15 files changed, 266 insertions(+), 265 deletions(-) diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index 363b79a44..9e6f27684 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -1105,7 +1105,7 @@ components: in: query description: The unit in which to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' cat.all_pit_segments::query.format: name: format in: query @@ -1164,7 +1164,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.allocation::query.cluster_manager_timeout: name: cluster_manager_timeout @@ -1374,7 +1374,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.fielddata::query.fields: in: query @@ -1503,7 +1503,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.indices::query.cluster_manager_timeout: name: cluster_manager_timeout @@ -1754,7 +1754,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.nodes::query.cluster_manager_timeout: name: cluster_manager_timeout @@ -1924,7 +1924,7 @@ components: in: query description: The unit in which to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' cat.pit_segments::query.format: name: format in: query @@ -2062,7 +2062,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.recovery::query.detailed: in: query @@ -2233,7 +2233,7 @@ components: in: query description: The unit in which to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' cat.segment_replication::query.completed_only: name: completed_only in: query @@ -2365,7 +2365,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.segments::query.cluster_manager_timeout: name: cluster_manager_timeout @@ -2444,7 +2444,7 @@ components: name: bytes description: The unit used to display byte values. schema: - $ref: '../schemas/_common.yaml#/components/schemas/StorageType' + $ref: '../schemas/_common.yaml#/components/schemas/ByteUnit' style: form cat.shards::query.cluster_manager_timeout: name: cluster_manager_timeout diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index e3e0926f4..b3cc12eb7 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2385,10 +2385,10 @@ components: items: $ref: '../schemas/indices.data_streams_stats.yaml#/components/schemas/DataStreamsStatsItem' total_store_sizes: - $ref: '../schemas/_common.yaml#/components/schemas/StorageSize' + $ref: '../schemas/_common.yaml#/components/schemas/HumanReadableByteCount' total_store_size_bytes: description: Total size, in bytes, of all shards for the selected data streams. - $ref: '../schemas/_common.yaml#/components/schemas/Bytes' + $ref: '../schemas/_common.yaml#/components/schemas/ByteCount' required: - _shards - backing_indices diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 39eca3980..9535014a0 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -725,10 +725,10 @@ components: $ref: '#/components/schemas/NodeId' NodeId: type: string - StorageSize: + HumanReadableByteCount: type: string pattern: '\d+(\.\d+)?(b|kb|k|mb|m|gb|g|tb|t|pb|p)' - StorageType: + ByteUnit: type: string enum: - b @@ -742,8 +742,9 @@ components: - pb - t - tb - Bytes: + ByteCount: type: integer + format: int64 Percentage: oneOf: - type: string @@ -928,9 +929,9 @@ components: properties: size_in_bytes: description: Total amount, in bytes, of memory used for completion across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' fields: type: object additionalProperties: @@ -941,9 +942,9 @@ components: type: object properties: size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - size_in_bytes DocStats: @@ -968,9 +969,9 @@ components: evictions: type: number memory_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' memory_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' fields: type: object additionalProperties: @@ -981,9 +982,9 @@ components: type: object properties: memory_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' memory_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - memory_size_in_bytes QueryCacheStats: @@ -1004,10 +1005,10 @@ components: description: Total count of query cache hits across all shards assigned to selected nodes. type: number memory_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' memory_size_in_bytes: description: Total amount, in bytes, of memory used for the query cache across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' miss_count: description: Total count of query cache misses across all shards assigned to selected nodes. type: number @@ -1074,22 +1075,22 @@ components: properties: failed: description: The number of bytes that failed to upload to the remote translog store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' failed_bytes: description: The number of bytes that failed to upload to the remote translog store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' started: description: The number of bytes successfully uploaded to the remote translog store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' started_bytes: description: The number of bytes successfully uploaded to the remote translog store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' succeeded: description: The number of bytes successfully uploaded to the remote translog store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' succeeded_bytes: description: The number of bytes successfully uploaded to the remote translog store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - failed_bytes - started_bytes @@ -1128,22 +1129,22 @@ components: properties: failed: description: The number of bytes that failed to upload/upload to/from the remote segment store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' failed_bytes: description: The number of bytes that failed to upload/upload to/from the remote segment store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' started: description: The number of bytes to upload/download to/from the remote segment store after the upload/download has started. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' started_bytes: description: The number of bytes to upload/download to/from the remote segment store after the upload/download has started. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' succeeded: description: The number of bytes successfully uploaded/downloaded to/from the remote segment store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' succeeded_bytes: description: The number of bytes successfully uploaded/downloaded to/from the remote segment store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - failed_bytes - started_bytes @@ -1163,16 +1164,16 @@ components: properties: max: description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' max_bytes: description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' total: description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' total_bytes: description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - max_bytes - total_bytes @@ -1196,11 +1197,11 @@ components: type: object properties: max_bytes_behind: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' max_replication_lag: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' total_bytes_behind: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - max_bytes_behind - max_replication_lag @@ -1212,10 +1213,10 @@ components: description: Total number of segments across all shards assigned to selected nodes. type: number doc_values_memory: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' doc_values_memory_in_bytes: description: Total amount, in bytes, of memory used for doc values across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' file_sizes: description: |- This object is not populated by the cluster stats API. @@ -1224,62 +1225,62 @@ components: additionalProperties: $ref: 'indices.stats.yaml#/components/schemas/ShardFileSizeInfo' fixed_bit_set: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' fixed_bit_set_memory_in_bytes: description: Total amount of memory, in bytes, used by fixed bit sets across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' index_writer_memory: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' index_writer_max_memory_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' index_writer_memory_in_bytes: description: Total amount, in bytes, of memory used by all index writers across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' max_unsafe_auto_id_timestamp: description: Unix timestamp, in milliseconds, of the most recently retried indexing request. type: number memory: description: Total amount, of memory used for segments across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' memory_in_bytes: description: Total amount, in bytes, of memory used for segments across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' norms_memory: description: Total amount of memory used for normalization factors across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' norms_memory_in_bytes: description: Total amount, in bytes, of memory used for normalization factors across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' points_memory: description: Total amount of memory used for points across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' points_memory_in_bytes: description: Total amount, in bytes, of memory used for points across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' stored_fields_memory: description: Total amount of memory used for stored fields across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' stored_fields_memory_in_bytes: description: Total amount, in bytes, of memory used for stored fields across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' terms_memory: description: Total amount of memory used for terms across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' terms_memory_in_bytes: description: Total amount, in bytes, of memory used for terms across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' term_vectors_memory: description: Total amount of memory used for term vectors across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' term_vectors_memory_in_bytes: description: Total amount, in bytes, of memory used for term vectors across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' version_map_memory: description: Total amount of memory used by all version maps across all shards assigned to selected nodes. - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' version_map_memory_in_bytes: description: Total amount, in bytes, of memory used by all version maps across all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' remote_store: $ref: '#/components/schemas/RemoteStoreStats' segment_replication: @@ -1302,15 +1303,15 @@ components: type: object properties: size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' size_in_bytes: description: Total size, in bytes, of all shards assigned to selected nodes. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' reserved: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' reserved_in_bytes: description: A prediction, in bytes, of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - reserved_in_bytes - size_in_bytes @@ -1624,21 +1625,21 @@ components: current_docs: type: number current_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' current_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' total: type: number total_auto_throttle: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' total_auto_throttle_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' total_docs: type: number total_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' total_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' total_stopped_time: $ref: '#/components/schemas/Duration' total_stopped_time_in_millis: @@ -1710,9 +1711,9 @@ components: hit_count: type: number memory_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' memory_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' miss_count: type: number required: @@ -1833,15 +1834,15 @@ components: remote_store: $ref: '#/components/schemas/RemoteStoreTranslogStats' size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' uncommitted_operations: type: number uncommitted_size: - $ref: '#/components/schemas/StorageSize' + $ref: '#/components/schemas/HumanReadableByteCount' uncommitted_size_in_bytes: - $ref: '#/components/schemas/Bytes' + $ref: '#/components/schemas/ByteCount' required: - earliest_last_modified_age - operations diff --git a/spec/schemas/cat.segments.yaml b/spec/schemas/cat.segments.yaml index 177c74e40..b0a119457 100644 --- a/spec/schemas/cat.segments.yaml +++ b/spec/schemas/cat.segments.yaml @@ -44,7 +44,7 @@ components: Also, OpenSearch creates extra deleted documents to internally track the recent history of operations on a shard. type: string size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' size.memory: type: string committed: diff --git a/spec/schemas/cluster.allocation_explain.yaml b/spec/schemas/cluster.allocation_explain.yaml index ec55c7cac..55d93622d 100644 --- a/spec/schemas/cluster.allocation_explain.yaml +++ b/spec/schemas/cluster.allocation_explain.yaml @@ -48,8 +48,8 @@ components: type: object additionalProperties: oneOf: - - $ref: '_common.yaml#/components/schemas/Bytes' - - $ref: '_common.yaml#/components/schemas/StorageSize' + - $ref: '_common.yaml#/components/schemas/ByteCount' + - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' shard_data_set_sizes: type: object additionalProperties: @@ -86,17 +86,17 @@ components: path: type: string total: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' used: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free_disk_percent: type: number used_disk_percent: @@ -188,7 +188,7 @@ components: in_sync: type: boolean matching_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' matching_sync_id: type: boolean store_exception: diff --git a/spec/schemas/cluster.stats.yaml b/spec/schemas/cluster.stats.yaml index 62e43b54f..d1590c7f7 100644 --- a/spec/schemas/cluster.stats.yaml +++ b/spec/schemas/cluster.stats.yaml @@ -213,10 +213,10 @@ components: description: Total number of fields in all non-system indices, accounting for mapping deduplication. type: number total_deduplicated_mapping_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_deduplicated_mapping_size_in_bytes: description: Total size of all mappings, in bytes, after deduplication and compression. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - field_types RuntimeFieldTypes: @@ -288,7 +288,7 @@ components: primary_shard_count: type: number total_primary_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' version: $ref: '_common.yaml#/components/schemas/VersionString' required: @@ -401,32 +401,32 @@ components: Total number of bytes available to JVM in file stores across all selected nodes. Depending on operating system or process-level restrictions, this number may be less than `nodes.fs.free_in_byes`. This is the actual amount of free disk space the selected OpenSearch nodes can use. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' available_in_bytes: description: |- Total number of bytes available to JVM in file stores across all selected nodes. Depending on operating system or process-level restrictions, this number may be less than `nodes.fs.free_in_byes`. This is the actual amount of free disk space the selected OpenSearch nodes can use. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: description: Total number of unallocated bytes in file stores across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_in_bytes: description: Total number of unallocated bytes in file stores across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total: description: Total size, of all file stores across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: description: Total size, in bytes, of all file stores across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' cache_reserved: description: Total size, of all cache reserved across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' x-version-added: '2.7' cache_reserved_in_bytes: description: Total size, in bytes, of all cache reserved across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' x-version-added: '2.7' required: - available_in_bytes @@ -445,7 +445,7 @@ components: current: $ref: '#/components/schemas/IndexingPressureMemorySummary' limit_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total: $ref: '#/components/schemas/IndexingPressureMemorySummary' required: @@ -456,19 +456,19 @@ components: type: object properties: all_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' combined_coordinating_and_primary_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' coordinating_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' coordinating_rejections: type: number primary_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' primary_rejections: type: number replica_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' replica_rejections: type: number required: @@ -534,16 +534,16 @@ components: properties: heap_max: description: Maximum amount of memory available for use by the heap across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_max_in_bytes: description: Maximum amount of memory, in bytes, available for use by the heap across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' heap_used: description: Memory currently in use by the heap across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_used_in_bytes: description: Memory, in bytes, currently in use by the heap across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - heap_max_in_bytes - heap_used_in_bytes @@ -645,28 +645,28 @@ components: properties: adjusted_total_in_bytes: description: Total amount, in bytes, of memory across all selected nodes, but using the value specified using the `es.total_memory_bytes` system property instead of measured total memory for those nodes where that system property was set. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: description: Amount, in bytes, of free physical memory across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_in_bytes: description: Amount, in bytes, of free physical memory across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free_percent: description: Percentage of free physical memory across all selected nodes. type: number total: description: Total amount of physical memory across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: description: Total amount, in bytes, of physical memory across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' used: description: Amount of physical memory in use across all selected nodes. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_in_bytes: description: Amount, in bytes, of physical memory in use across all selected nodes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' used_percent: description: Percentage of physical memory in use across all selected nodes. type: number diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index ea02f12f8..90d6776d6 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -382,7 +382,7 @@ components: durability: $ref: '#/components/schemas/TranslogDurability' flush_threshold_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' retention: $ref: '#/components/schemas/TranslogRetention' TranslogDurability: @@ -394,7 +394,7 @@ components: type: object properties: size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' age: $ref: '_common.yaml#/components/schemas/Duration' SettingsQueryString: @@ -787,13 +787,13 @@ components: max_docs: type: number min_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' min_primary_shard_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_primary_shard_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' min_primary_shard_docs: type: number max_primary_shard_docs: diff --git a/spec/schemas/indices.data_streams_stats.yaml b/spec/schemas/indices.data_streams_stats.yaml index aec96d68b..f8309bb2e 100644 --- a/spec/schemas/indices.data_streams_stats.yaml +++ b/spec/schemas/indices.data_streams_stats.yaml @@ -17,10 +17,10 @@ components: maximum_timestamp: $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' store_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' store_size_bytes: description: Total size, in bytes, of all shards for the data stream's backing indices. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - backing_indices - data_stream diff --git a/spec/schemas/indices.recovery.yaml b/spec/schemas/indices.recovery.yaml index c0d455478..a4f6eaeec 100644 --- a/spec/schemas/indices.recovery.yaml +++ b/spec/schemas/indices.recovery.yaml @@ -95,21 +95,21 @@ components: percent: $ref: '_common.yaml#/components/schemas/Percentage' recovered: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' recovered_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' recovered_from_snapshot: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' recovered_from_snapshot_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' reused: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' reused_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - percent - recovered_in_bytes diff --git a/spec/schemas/indices.rollover.yaml b/spec/schemas/indices.rollover.yaml index 05e73c318..f78347fe4 100644 --- a/spec/schemas/indices.rollover.yaml +++ b/spec/schemas/indices.rollover.yaml @@ -20,21 +20,21 @@ components: max_docs: type: number max_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_size_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' min_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' min_size_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max_primary_shard_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_primary_shard_size_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' min_primary_shard_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' min_primary_shard_size_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max_primary_shard_docs: type: number min_primary_shard_docs: diff --git a/spec/schemas/indices.segments.yaml b/spec/schemas/indices.segments.yaml index c049e3ef6..ae2c62292 100644 --- a/spec/schemas/indices.segments.yaml +++ b/spec/schemas/indices.segments.yaml @@ -60,13 +60,13 @@ components: deleted_docs: type: integer size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' memory: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' memory_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' committed: type: boolean search: diff --git a/spec/schemas/indices.stats.yaml b/spec/schemas/indices.stats.yaml index 6a12537b2..b2391cddb 100644 --- a/spec/schemas/indices.stats.yaml +++ b/spec/schemas/indices.stats.yaml @@ -12,15 +12,15 @@ components: description: type: string size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' min_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' average_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' count: type: number required: diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index eb0c14f85..51fb8b7cf 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -56,7 +56,7 @@ components: type: number request_size_bytes: description: Cumulative size in bytes of all requests from this client. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' x_opaque_id: description: |- Value from the client's `x-opaque-id` HTTP header. @@ -388,9 +388,9 @@ components: type: object properties: size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' evictions: type: number hit_count: @@ -428,16 +428,16 @@ components: properties: estimated_size: description: Estimated memory used for the operation. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' estimated_size_in_bytes: description: Estimated memory used, in bytes, for the operation. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' limit_size: description: Memory limit for the circuit breaker. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' limit_size_in_bytes: description: Memory limit, in bytes, for the circuit breaker. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' overhead: description: A constant that all estimates for the circuit breaker are multiplied with to calculate a final estimate. type: number @@ -466,38 +466,38 @@ components: properties: available: description: Total amount of disk space available to this Java virtual machine on this file store. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' available_in_bytes: description: Total number of bytes available to this Java virtual machine on this file store. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' cache_reserved: x-version-added: '2.7' description: Total number of cache bytes reserved available to this Java virtual machine on this file store. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' cache_reserved_in_bytes: x-version-added: '2.7' description: Total number of cache bytes reserved available to this Java virtual machine on this file store. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' disk_queue: type: string disk_reads: type: number disk_read_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' disk_read_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' disk_writes: type: number disk_write_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' disk_write_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: description: Total amount of unallocated disk space in the file store. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_in_bytes: description: Total number of unallocated bytes in the file store. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' mount: description: 'Mount point of the file store (for example: `/dev/sda2`).' type: string @@ -506,10 +506,10 @@ components: type: string total: description: Total size of the file store. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: description: Total size of the file store in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' type: description: 'Type of the file store (ex: ext4).' type: string @@ -521,33 +521,33 @@ components: Total disk space available to this Java virtual machine on all file stores. Depending on OS or process level restrictions, this might appear less than `free`. This is the actual amount of free disk space the OpenSearch node can utilize. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' available_in_bytes: description: |- Total number of bytes available to this Java virtual machine on all file stores. Depending on OS or process level restrictions, this might appear less than `free_in_bytes`. This is the actual amount of free disk space the OpenSearch node can utilize. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: description: Total unallocated disk space in all file stores. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_in_bytes: description: Total number of unallocated bytes in all file stores. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total: description: Total size of all file stores. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: description: Total size of all file stores in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' cache_reserved: x-version-added: '2.7' description: Total size of cache bytes reserved in all file stores. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' cache_reserved_in_bytes: x-version-added: '2.7' description: Total size of cache bytes reserved in all file stores. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' IoStats: type: object properties: @@ -624,16 +624,16 @@ components: type: number total_capacity: description: Total capacity of buffer pools. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_capacity_in_bytes: description: Total capacity of buffer pools in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' used: description: Size of buffer pools. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_in_bytes: description: Size of buffer pools in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' JvmClasses: type: object properties: @@ -671,37 +671,37 @@ components: properties: heap_used: description: Memory currently in use by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_used_in_bytes: description: Memory, in bytes, currently in use by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' heap_used_percent: description: Percentage of memory currently in use by the heap. type: number heap_committed: description: Amount of memory available for use by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_committed_in_bytes: description: Amount of memory, in bytes, available for use by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' heap_max: description: Maximum amount of memory available for use by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_max_in_bytes: description: Maximum amount of memory, in bytes, available for use by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' non_heap_used: description: Non-heap memory used. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' non_heap_used_in_bytes: description: Non-heap memory used, in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' non_heap_committed: description: Amount of non-heap memory available. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' non_heap_committed_in_bytes: description: Amount of non-heap memory available, in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' pools: description: Contains statistics about heap memory usage for the node. type: object @@ -712,28 +712,28 @@ components: properties: used: description: Memory used by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_in_bytes: description: Memory, in bytes, used by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max: description: Maximum amount of memory available for use by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_in_bytes: description: Maximum amount of memory, in bytes, available for use by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' peak_used: description: Largest amount of memory historically used by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' peak_used_in_bytes: description: Largest amount of memory, in bytes, historically used by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' peak_max: description: Largest amount of memory historically used by the heap. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' peak_max_in_bytes: description: Largest amount of memory, in bytes, historically used by the heap. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' last_gc_stats: description: Most recent GC cycle stats for a particular memory pool. $ref: '#/components/schemas/LastGcStats' @@ -741,13 +741,13 @@ components: type: object properties: used: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' usage_percent: type: number JvmThreads: @@ -811,37 +811,37 @@ components: description: |- If the amount of physical memory has been overridden using the `es`.`total_memory_bytes` system property then this reports the overridden value in bytes. Otherwise it reports the same value as `total_in_bytes`. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' resident: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' resident_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' share: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' share_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total_virtual: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_virtual_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total: description: Total amount of physical memory. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: description: Total amount of physical memory in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' free: description: Amount of free physical memory. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' free_in_bytes: description: Amount of free physical memory in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' used: description: Amount of used physical memory. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' used_in_bytes: description: Amount of used physical memory in bytes. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' Cgroup: type: object properties: @@ -951,10 +951,10 @@ components: type: number rx_size: description: Size of RX packets received by the node during internal cluster communication. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' rx_size_in_bytes: description: Size, in bytes, of RX packets received by the node during internal cluster communication. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' server_open: description: Current number of inbound TCP connections used for internal communication between nodes. type: number @@ -963,10 +963,10 @@ components: type: number tx_size: description: Size of TX packets sent by the node during internal cluster communication. - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' tx_size_in_bytes: description: Size, in bytes, of TX packets sent by the node during internal cluster communication. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' total_outbound_connections: description: |- The cumulative number of outbound transport connections that this node has opened since it started. @@ -1096,13 +1096,13 @@ components: count: type: number uncompressed_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' uncompressed_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' compressed_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' compressed_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' ClusterAppliedStats: type: object properties: @@ -1179,17 +1179,17 @@ components: cancellation_count: type: number current_max: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' current_max_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' current_avg: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' current_avg_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' rolling_avg: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' rolling_avg_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' ShardSearchBackpressureTaskResourceTrackerElapsedTimeTrackerStats: type: object properties: @@ -1240,12 +1240,12 @@ components: type: object properties: limit: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' limit_in_bytes: description: |- Configured memory limit, in bytes, for the indexing requests. Replica requests have an automatic limit that is 1.5x this value. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' current: $ref: '#/components/schemas/PressureMemory' total: @@ -1254,32 +1254,32 @@ components: type: object properties: all: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' all_in_bytes: description: Memory consumed, in bytes, by indexing requests in the coordinating, primary, or replica stage. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' combined_coordinating_and_primary: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' combined_coordinating_and_primary_in_bytes: description: |- Memory consumed, in bytes, by indexing requests in the coordinating or primary stage. This value is not the sum of coordinating and primary as a node can reuse the coordinating memory if the primary stage is executed locally. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' coordinating: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' coordinating_in_bytes: description: Memory consumed, in bytes, by indexing requests in the coordinating stage. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' primary: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' primary_in_bytes: description: Memory consumed, in bytes, by indexing requests in the primary stage. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' replica: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' replica_in_bytes: description: Memory consumed, in bytes, by indexing requests in the replica stage. - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' coordinating_rejections: description: Number of indexing requests rejected in the coordinating stage. type: number diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7102b54b9..2d27da447 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -82,7 +82,7 @@ components: description: Total heap allowed to be used to hold recently indexed documents before they must be written to disk. This size is a shared pool across all shards on this node, and is controlled by Indexing Buffer settings. type: number total_indexing_buffer_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' transport: $ref: '#/components/schemas/NodeInfoTransport' transport_address: @@ -115,9 +115,9 @@ components: items: type: string max_content_length: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' max_content_length_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' publish_address: type: string required: @@ -170,25 +170,25 @@ components: type: object properties: direct_max: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' direct_max_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' heap_init: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_init_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' heap_max: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' heap_max_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' non_heap_init: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' non_heap_init_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' non_heap_max: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' non_heap_max_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - direct_max_in_bytes - heap_init_in_bytes @@ -251,9 +251,9 @@ components: type: object properties: cache_size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' cache_size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' cores_per_socket: type: number mhz: @@ -279,9 +279,9 @@ components: type: object properties: total: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' total_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - total - total_in_bytes diff --git a/spec/schemas/snapshot._common.yaml b/spec/schemas/snapshot._common.yaml index f00611ec7..83c8a8d9c 100644 --- a/spec/schemas/snapshot._common.yaml +++ b/spec/schemas/snapshot._common.yaml @@ -89,9 +89,9 @@ components: shard_count: type: number size: - $ref: '_common.yaml#/components/schemas/StorageSize' + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' max_segments_per_shard: type: number required: @@ -228,7 +228,7 @@ components: file_count: type: number size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - file_count - size_in_bytes @@ -278,7 +278,7 @@ components: file_count: type: number size_in_bytes: - $ref: '_common.yaml#/components/schemas/Bytes' + $ref: '_common.yaml#/components/schemas/ByteCount' required: - file_count - size_in_bytes From 97cfa9b129908332e7135dfac2237db6b2fd5469 Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Thu, 12 Sep 2024 08:52:48 -0700 Subject: [PATCH 15/73] fix matchAllQuery (#572) --- spec/schemas/_common.query_dsl.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index c4852c848..c98fa14f5 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -984,9 +984,7 @@ components: - all - none MatchAllQuery: - allOf: - - $ref: '#/components/schemas/QueryBase' - - type: object + $ref: '#/components/schemas/QueryBase' MatchBoolPrefixQuery: allOf: - $ref: '#/components/schemas/QueryBase' From 584d5251d9359b0a9131cd9cb884b3fa3bc9ce2d Mon Sep 17 00:00:00 2001 From: Jakob Date: Fri, 13 Sep 2024 12:41:57 +0200 Subject: [PATCH 16/73] Split `Percentage` into `PercentageNumber`, `PercentageString` (#573) Signed-off-by: Jakob Hahn --- CHANGELOG.md | 1 + spec/schemas/_common.yaml | 9 +++++---- spec/schemas/cat._common.yaml | 4 ++-- spec/schemas/cat.allocation.yaml | 2 +- spec/schemas/cat.health.yaml | 2 +- spec/schemas/cat.nodes.yaml | 8 ++++---- spec/schemas/cat.recovery.yaml | 6 +++--- spec/schemas/cluster.allocation_explain.yaml | 4 ++-- spec/schemas/cluster.health.yaml | 4 +++- spec/schemas/cluster.stats.yaml | 6 +++--- spec/schemas/indices.recovery.yaml | 6 +++--- spec/schemas/nodes._common.yaml | 20 +++++++++++--------- tests/default/cluster/health.yaml | 1 + 13 files changed, 40 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 002280bae..930050050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Corrected Sort schema for `/_search`'s request body ([#529](https://github.com/opensearch-project/opensearch-api-specification/issues/529)) - Rename `Bytes` component to `StorageType` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Rename `ByteSize` to `StorageSize` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) +- Split `Percentage` into `PercentageNumber`, `PercentageString` ([#573](https://github.com/opensearch-project/opensearch-api-specification/pull/573)) ### Deprecated diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 9535014a0..140600787 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -745,10 +745,11 @@ components: ByteCount: type: integer format: int64 - Percentage: - oneOf: - - type: string - - type: number + PercentageNumber: + type: number + PercentageString: + type: string + pattern: '\d+(\.\d+)?' Host: type: string Ip: diff --git a/spec/schemas/cat._common.yaml b/spec/schemas/cat._common.yaml index d455d4376..0eeda5f43 100644 --- a/spec/schemas/cat._common.yaml +++ b/spec/schemas/cat._common.yaml @@ -63,11 +63,11 @@ components: files_fetched: type: string files_percent: - type: string + $ref: '_common.yaml#/components/schemas/PercentageString' bytes_fetched: type: string bytes_percent: - type: string + $ref: '_common.yaml#/components/schemas/PercentageString' start_time: type: string stop_time: diff --git a/spec/schemas/cat.allocation.yaml b/spec/schemas/cat.allocation.yaml index ba97dba01..6b499fb89 100644 --- a/spec/schemas/cat.allocation.yaml +++ b/spec/schemas/cat.allocation.yaml @@ -36,7 +36,7 @@ components: disk.percent: description: Total percentage of disk space in use. Calculated as `disk.used / disk.total`. oneOf: - - $ref: '_common.yaml#/components/schemas/Percentage' + - $ref: '_common.yaml#/components/schemas/PercentageString' - type: 'null' host: description: Network host for the node. Set using the `network.host` setting. diff --git a/spec/schemas/cat.health.yaml b/spec/schemas/cat.health.yaml index 6e7afc949..d0b61ad0d 100644 --- a/spec/schemas/cat.health.yaml +++ b/spec/schemas/cat.health.yaml @@ -48,7 +48,7 @@ components: type: string active_shards_percent: description: active number of shards in percent - type: string + $ref: '_common.yaml#/components/schemas/PercentageString' discovered_cluster_manager: description: cluster manager is discovered or not type: string diff --git a/spec/schemas/cat.nodes.yaml b/spec/schemas/cat.nodes.yaml index 80109fedf..7187d5c14 100644 --- a/spec/schemas/cat.nodes.yaml +++ b/spec/schemas/cat.nodes.yaml @@ -44,12 +44,12 @@ components: disk.avail: type: string disk.used_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' heap.current: description: The used heap. type: string heap.percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' heap.max: description: The maximum configured heap. type: string @@ -57,7 +57,7 @@ components: description: The used machine memory. type: string ram.percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' ram.max: description: The total machine memory. type: string @@ -65,7 +65,7 @@ components: description: The used file descriptors. type: string file_desc.percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' file_desc.max: description: The maximum number of file descriptors. type: string diff --git a/spec/schemas/cat.recovery.yaml b/spec/schemas/cat.recovery.yaml index ec885f558..947dab27a 100644 --- a/spec/schemas/cat.recovery.yaml +++ b/spec/schemas/cat.recovery.yaml @@ -55,7 +55,7 @@ components: description: The files recovered. type: string files_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' files_total: description: The total number of files. type: string @@ -66,7 +66,7 @@ components: description: The bytes recovered. type: string bytes_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' bytes_total: description: The total number of bytes. type: string @@ -77,4 +77,4 @@ components: description: The translog operations recovered. type: string translog_ops_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' diff --git a/spec/schemas/cluster.allocation_explain.yaml b/spec/schemas/cluster.allocation_explain.yaml index 55d93622d..f7141a328 100644 --- a/spec/schemas/cluster.allocation_explain.yaml +++ b/spec/schemas/cluster.allocation_explain.yaml @@ -98,9 +98,9 @@ components: free_bytes: $ref: '_common.yaml#/components/schemas/ByteCount' free_disk_percent: - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' used_disk_percent: - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' required: - free_bytes - free_disk_percent diff --git a/spec/schemas/cluster.health.yaml b/spec/schemas/cluster.health.yaml index 9e94db0dc..3af259176 100644 --- a/spec/schemas/cluster.health.yaml +++ b/spec/schemas/cluster.health.yaml @@ -15,8 +15,10 @@ components: active_shards: description: The total number of active primary and replica shards. type: number + active_shards_percent: + $ref: '_common.yaml#/components/schemas/PercentageString' active_shards_percent_as_number: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageNumber' awareness_attributes: description: Cluster health information for each awareness attribute. type: object diff --git a/spec/schemas/cluster.stats.yaml b/spec/schemas/cluster.stats.yaml index d1590c7f7..cd6ff6864 100644 --- a/spec/schemas/cluster.stats.yaml +++ b/spec/schemas/cluster.stats.yaml @@ -654,7 +654,7 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' free_percent: description: Percentage of free physical memory across all selected nodes. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' total: description: Total amount of physical memory across all selected nodes. $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' @@ -669,7 +669,7 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' used_percent: description: Percentage of physical memory in use across all selected nodes. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' required: - free_in_bytes - free_percent @@ -730,7 +730,7 @@ components: description: |- Percentage of CPU used across all selected nodes. Returns `-1` if not supported. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' required: - percent ClusterProcessOpenFileDescriptors: diff --git a/spec/schemas/indices.recovery.yaml b/spec/schemas/indices.recovery.yaml index a4f6eaeec..d2b781830 100644 --- a/spec/schemas/indices.recovery.yaml +++ b/spec/schemas/indices.recovery.yaml @@ -93,7 +93,7 @@ components: type: object properties: percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' recovered: $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' recovered_in_bytes: @@ -123,7 +123,7 @@ components: items: $ref: '#/components/schemas/FileDetails' percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' recovered: type: number reused: @@ -193,7 +193,7 @@ components: type: object properties: percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' recovered: type: number total: diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 51fb8b7cf..9ba3c0621 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -114,6 +114,8 @@ components: failed: description: Number of failed operations for the processor. type: number + time: + $ref: '_common.yaml#/components/schemas/Duration' time_in_millis: $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' ThreadCount: @@ -346,16 +348,16 @@ components: timestamp: type: number cpu_utilization_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' memory_utilization_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' io_usage_stats: $ref: '#/components/schemas/ShardResourceUsageStatsIoUsageStats' ShardResourceUsageStatsIoUsageStats: type: object properties: max_io_utilization_percent: - $ref: '_common.yaml#/components/schemas/Percentage' + $ref: '_common.yaml#/components/schemas/PercentageString' ShardSegmentReplicationBackpressureStats: type: object properties: @@ -677,7 +679,7 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' heap_used_percent: description: Percentage of memory currently in use by the heap. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' heap_committed: description: Amount of memory available for use by the heap. $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' @@ -749,7 +751,7 @@ components: max_in_bytes: $ref: '_common.yaml#/components/schemas/ByteCount' usage_percent: - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' JvmThreads: type: object properties: @@ -776,7 +778,7 @@ components: type: object properties: percent: - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' sys: $ref: '_common.yaml#/components/schemas/Duration' sys_in_millis: @@ -800,10 +802,10 @@ components: properties: free_percent: description: Percentage of free memory. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' used_percent: description: Percentage of used memory. - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' MemoryStats: type: object properties: @@ -1224,7 +1226,7 @@ components: cancellation_limit_reached_count: type: number cancelled_task_percentage: - type: number + $ref: '_common.yaml#/components/schemas/PercentageNumber' current_cancellation_eligible_tasks_count: type: number TotalRejectionsBreakup: diff --git a/tests/default/cluster/health.yaml b/tests/default/cluster/health.yaml index 3d00ab376..7453963a0 100644 --- a/tests/default/cluster/health.yaml +++ b/tests/default/cluster/health.yaml @@ -9,6 +9,7 @@ chapters: expand_wildcards: all level: cluster local: false + human: true timeout: 10s wait_for_active_shards: 0 wait_for_nodes: <2 From c2c666fda9c03fddd309253d6c96cfd1b3bfe917 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Fri, 13 Sep 2024 13:01:01 -0400 Subject: [PATCH 17/73] Fixed/added ISM APIs. (#568) --- CHANGELOG.md | 2 + spec/namespaces/ism.yaml | 229 ++++++++++++++++ spec/namespaces/transforms.yaml | 14 +- spec/schemas/ism._common.yaml | 376 ++++++++++++++++++++++++++ tests/default/ism/add.yaml | 39 +++ tests/default/ism/change_policy.yaml | 47 ++++ tests/default/ism/explain.yaml | 40 +++ tests/default/ism/policies.yaml | 81 ++++++ tests/default/ism/remove.yaml | 45 +++ tests/default/transforms/preview.yaml | 43 +++ 10 files changed, 915 insertions(+), 1 deletion(-) create mode 100644 spec/namespaces/ism.yaml create mode 100644 spec/schemas/ism._common.yaml create mode 100644 tests/default/ism/add.yaml create mode 100644 tests/default/ism/change_policy.yaml create mode 100644 tests/default/ism/explain.yaml create mode 100644 tests/default/ism/policies.yaml create mode 100644 tests/default/ism/remove.yaml create mode 100644 tests/default/transforms/preview.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 930050050..d91649a66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Added `/_plugins/_flow_framework/` ,`_search`,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) +- Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ### Changed @@ -154,6 +155,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561)) - Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) +- Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ### Security diff --git a/spec/namespaces/ism.yaml b/spec/namespaces/ism.yaml new file mode 100644 index 000000000..3aa5688f4 --- /dev/null +++ b/spec/namespaces/ism.yaml @@ -0,0 +1,229 @@ +openapi: 3.1.0 +info: + title: OpenSearch ISM API + description: Use the index state management operations to programmatically work with policies and managed indexes. + version: 1.0.0 +paths: + /_plugins/_ism/policies/{policy_id}: + get: + operationId: ism.get_policy.0 + x-operation-group: ism.get_policy + description: Gets the policy. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#get-policy + parameters: + - $ref: '#/components/parameters/ism.get_policy::path.policy_id' + responses: + '200': + $ref: '#/components/responses/ism.get_policy@200' + put: + operationId: ism.put_policy.0 + x-operation-group: ism.put_policy + description: Creates or updates a policy. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#create-policy + parameters: + - $ref: '#/components/parameters/ism.put_policy::path.policy_id' + - $ref: '#/components/parameters/ism.put_policy::query.if_primary_term' + - $ref: '#/components/parameters/ism.put_policy::query.if_seq_no' + requestBody: + $ref: '#/components/requestBodies/ism.put_policy' + responses: + '201': + $ref: '#/components/responses/ism.put_policy@201' + '200': + $ref: '#/components/responses/ism.put_policy@200' + delete: + operationId: ism.delete_policy.0 + x-operation-group: ism.delete_policy + description: Deletes a policy. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#delete-policy + parameters: + - $ref: '#/components/parameters/ism.delete_policy::path.policy_id' + responses: + '200': + $ref: '#/components/responses/ism.delete_policy@200' + /_plugins/_ism/add/{index}: + post: + operationId: ism.add_policy.0 + x-operation-group: ism.add_policy + description: Adds a policy to an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#add-policy + parameters: + - $ref: '#/components/parameters/ism.add_policy::path.index' + requestBody: + $ref: '#/components/requestBodies/ism.add_policy' + responses: + '200': + $ref: '#/components/responses/ism.add_policy@200' + /_plugins/_ism/remove/{index}: + post: + operationId: ism.remove_policy.0 + x-operation-group: ism.remove_policy + description: Removes a policy from an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#remove-policy + parameters: + - $ref: '#/components/parameters/ism.remove_policy::path.index' + responses: + '200': + $ref: '#/components/responses/ism.remove_policy@200' + /_plugins/_ism/change_policy/{index}: + post: + operationId: ism.change_policy.0 + x-operation-group: ism.change_policy + description: Updates the managed index policy to a new policy. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#update-managed-index-policy + parameters: + - $ref: '#/components/parameters/ism.change_policy::path.index' + requestBody: + $ref: '#/components/requestBodies/ism.change_policy' + responses: + '200': + $ref: '#/components/responses/ism.change_policy@200' + /_plugins/_ism/explain/{index}: + get: + operationId: ism.explain_index.0 + x-operation-group: ism.explain_index + description: Gets the currently applied policy on an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#explain-index + parameters: + - $ref: '#/components/parameters/ism.explain_index::path.index' + responses: + '200': + $ref: '#/components/responses/ism.explain_index@200' +components: + requestBodies: + ism.put_policy: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyRequest' + ism.add_policy: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/AddPolicyRequest' + ism.change_policy: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/ChangePolicyRequest' + responses: + ism.get_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/GetPolicyResponse' + ism.put_policy@201: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyResponse' + ism.put_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyResponse' + ism.delete_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/DeletePolicyResponse' + ism.add_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/AddPolicyResponse' + ism.remove_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/RemovePolicyResponse' + ism.change_policy@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/ChangePolicyResponse' + ism.explain_index@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/ExplainIndexResponse' + parameters: + ism.get_policy::path.policy_id: + name: policy_id + in: path + required: true + schema: + type: string + ism.put_policy::path.policy_id: + name: policy_id + in: path + required: true + schema: + type: string + ism.delete_policy::path.policy_id: + name: policy_id + in: path + required: true + schema: + type: string + ism.put_policy::query.if_seq_no: + name: if_seq_no + in: query + description: Only perform the operation if the document has this sequence number. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/SequenceNumber' + style: form + ism.put_policy::query.if_primary_term: + name: if_primary_term + in: query + description: Only perform the operation if the document has this primary term. + schema: + type: number + style: form + ism.add_policy::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + ism.remove_policy::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + ism.change_policy::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + ism.explain_index::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple diff --git a/spec/namespaces/transforms.yaml b/spec/namespaces/transforms.yaml index b525f101a..00c3fd4e6 100644 --- a/spec/namespaces/transforms.yaml +++ b/spec/namespaces/transforms.yaml @@ -102,18 +102,30 @@ paths: '200': $ref: '#/components/responses/transforms.explain@200' /_plugins/_transform/_preview: - get: + post: operationId: transforms.preview.0 x-operation-group: transforms.preview x-version-added: '1.0' description: Returns a preview of what a transformed index would look like. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#preview-a-transform-jobs-results + requestBody: + $ref: '#/components/requestBodies/transforms.preview' responses: '200': $ref: '#/components/responses/transforms.preview@200' components: requestBodies: + transforms.preview: + content: + application/json: + schema: + type: object + properties: + transform: + $ref: '../schemas/transforms._common.yaml#/components/schemas/Transform' + required: + - transform transforms.put: content: application/json: diff --git a/spec/schemas/ism._common.yaml b/spec/schemas/ism._common.yaml new file mode 100644 index 000000000..19e0762ce --- /dev/null +++ b/spec/schemas/ism._common.yaml @@ -0,0 +1,376 @@ +openapi: 3.1.0 +info: + title: Schemas of ism._common Category + description: Schemas of ism._common category. + version: 1.0.0 +paths: {} +components: + schemas: + Metadata: + type: object + properties: + _id: + $ref: '_common.yaml#/components/schemas/Id' + _primary_term: + type: number + _seq_no: + $ref: '_common.yaml#/components/schemas/SequenceNumber' + _version: + $ref: '_common.yaml#/components/schemas/VersionNumber' + Policy: + type: object + description: |- + A policy defines how an index is managed throughout its lifetime. + properties: + policy_id: + type: string + description: + type: string + last_updated_time: + type: integer # TODO: field is strict_date_time||epoch_millis + schema_version: + type: number + error_notification: + oneOf: + - $ref: '#/components/schemas/ErrorNotification' + - type: 'null' + default_state: + type: string + description: The default state an index is in. + states: + type: array + description: List of states for the index to transition between. + items: + $ref: '#/components/schemas/States' + ism_template: + oneOf: + - $ref: '#/components/schemas/IsmTemplate' + - type: 'null' + - type: array + items: + $ref: '#/components/schemas/IsmTemplate' + States: + type: object + description: |- + A list of actions to perform, and transitions to enter a new state. + Once a managed index enters a state it will sequentially execute the actions + in the same order listed in the policy. Once all actions have been successfully completed + state transitions will be checked until a true condition is eventually met. + If you define multiple transitions in a state, the first one in the list that is true will be used. + properties: + name: + type: string + actions: + type: array + description: A list of actions to perform. + items: + $ref: '#/components/schemas/Action' + transitions: + type: array + description: A list of transitions to enter a new state. + items: + $ref: '#/components/schemas/Transition' + Action: + type: object + description: An action to perform. + properties: + timeout: + $ref: '#/components/schemas/ActionTimeout' + retry: + $ref: '#/components/schemas/ActionRetry' + alias: + $ref: '#/components/schemas/ActionAlias' + delete: + $ref: '#/components/schemas/ActionDelete' + force_merge: + $ref: '#/components/schemas/ActionForceMerge' + read_only: + $ref: '#/components/schemas/ActionReadOnly' + read_write: + $ref: '#/components/schemas/ActionReadWrite' + replica_count: + $ref: '#/components/schemas/ActionReplicaCount' + index_priority: + $ref: '#/components/schemas/ActionIndexPriority' + close: + $ref: '#/components/schemas/ActionClose' + open: + $ref: '#/components/schemas/ActionOpen' + snapshot: + $ref: '#/components/schemas/ActionSnapshot' + rollover: + $ref: '#/components/schemas/ActionRollover' + notification: + $ref: '#/components/schemas/ActionNotification' + allocation: + $ref: '#/components/schemas/ActionAllocation' + rollup: + $ref: '#/components/schemas/ActionRollup' + transform: + $ref: '#/components/schemas/ActionTransform' + shrink: + $ref: '#/components/schemas/ActionShrink' + custom: + $ref: '#/components/schemas/ActionCustom' + ActionTimeout: + type: object + properties: + timeout: + type: object + ActionRetry: + type: object + properties: + count: + type: integer + backoff: + type: string + delay: + type: string + ActionAlias: + type: object + properties: + actions: + type: object + additionalProperties: true + ActionDelete: + type: object + ActionForceMerge: + type: object + properties: + max_num_segments: + type: integer + ActionReadOnly: + type: object + ActionReadWrite: + type: object + ActionReplicaCount: + type: object + properties: + number_of_replicas: + type: number + ActionIndexPriority: + type: object + properties: + priority: + type: number + ActionClose: + type: object + ActionOpen: + type: object + ActionSnapshot: + type: object + properties: + repository: + type: string + snapshot: + type: string + include_global_state: + type: boolean + ActionRollover: + type: object + properties: + min_size: + type: number + min_index_age: + type: number + min_doc_count: + type: number + min_primary_shard_size: + type: number + copy_alias: + type: boolean + ActionNotification: + type: object + additionalProperties: true + ActionAllocation: + type: object + properties: + require: + type: object + additionalProperties: true + include: + type: object + additionalProperties: true + exclude: + type: object + additionalProperties: true + wait_for: + type: boolean + ActionRollup: + type: object + additionalProperties: true + ActionTransform: + type: object + additionalProperties: true + ActionShrink: + type: object + additionalProperties: true + ActionCustom: + type: object + additionalProperties: true + Transition: + type: object + description: A transition into a new state. + properties: + state_name: + type: string + conditions: + type: object + additionalProperties: true + GetPolicyResponse: + allOf: + - $ref: '#/components/schemas/Metadata' + - $ref: '#/components/schemas/PolicyEnvelope' + PutPolicyResponse: + allOf: + - $ref: '#/components/schemas/Metadata' + - type: object + properties: + policy: + $ref: '#/components/schemas/PolicyEnvelope' + DeletePolicyResponse: + $ref: '_common.yaml#/components/schemas/WriteResponseBase' + ChangePolicyResponse: + type: object + properties: + updated_indices: + type: number + failures: + type: boolean + failed_indices: + type: array + items: + $ref: '#/components/schemas/FailedIndex' + FailedIndex: + type: object + properties: + index_name: + type: string + index_uuid: + type: string + reason: + type: string + RemovePolicyResponse: + $ref: '#/components/schemas/ChangePolicyResponse' + AddPolicyResponse: + $ref: '#/components/schemas/ChangePolicyResponse' + PolicyEnvelope: + type: object + properties: + policy: + $ref: '#/components/schemas/Policy' + PutPolicyRequest: + $ref: '#/components/schemas/PolicyEnvelope' + AddPolicyRequest: + type: object + properties: + policy_id: + type: string + required: + - policy_id + ChangePolicyRequest: + type: object + properties: + policy_id: + type: string + state: + type: string + include: + type: array + items: + $ref: '#/components/schemas/IncludeState' + required: + - policy_id + IncludeState: + type: object + properties: + state: + type: string + ExplainIndexResponse: + type: object + properties: + total_managed_indices: + type: number + additionalProperties: + $ref: '#/components/schemas/ExplainPolicy' + ExplainPolicy: + type: object + properties: + index.plugins.index_state_management.policy_id: + type: ['null', string] + index.opendistro.index_state_management.policy_id: + type: ['null', string] + enabled: + type: [boolean, 'null'] + ErrorNotification: + type: object + properties: + destination: + $ref: '#/components/schemas/ErrorNotificationDestination' + channel: + $ref: '#/components/schemas/Channel' + message_template: + type: object + additionalProperties: true + Channel: + type: object + properties: + id: + type: string + ErrorNotificationDestination: + type: object + properties: + name: + type: string + last_update_time: + type: integer + additionalProperties: + anyOf: + - $ref: '#/components/schemas/ErrorNotificationChime' + - $ref: '#/components/schemas/ErrorNotificationSlack' + ErrorNotificationChime: + type: object + properties: + url: + type: string + ErrorNotificationSlack: + type: object + properties: + url: + type: string + custom_webhook: + $ref: '#/components/schemas/SlackCustomWebhook' + SlackCustomWebhook: + type: object + properties: + url: + type: string + scheme: + type: string + host: + type: string + port: + type: integer + path: + type: string + query_params: + type: object + additionalProperties: true + header_params: + type: object + additionalProperties: true + username: + type: string + password: + type: string + IsmTemplate: + type: object + properties: + index_patterns: + type: array + items: + type: string + priority: + type: number + last_updated_time: + type: integer diff --git a/tests/default/ism/add.yaml b/tests/default/ism/add.yaml new file mode 100644 index 000000000..f5283aa96 --- /dev/null +++ b/tests/default/ism/add.yaml @@ -0,0 +1,39 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test adding a policy to an index. +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +prologues: + - path: /books + method: PUT + - id: policy + path: /_plugins/_ism/policies/rollover + method: PUT + request: + payload: + policy: + description: A rollover policy. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 5 + status: [201] + output: + id: payload._id +chapters: + - synopsis: Add a rollover policy to the books index. + path: /_plugins/_ism/add/{index} + method: POST + parameters: + index: books + request: + payload: + policy_id: ${policy.id} + \ No newline at end of file diff --git a/tests/default/ism/change_policy.yaml b/tests/default/ism/change_policy.yaml new file mode 100644 index 000000000..3268f0d0a --- /dev/null +++ b/tests/default/ism/change_policy.yaml @@ -0,0 +1,47 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test updating a managed index policy. +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +prologues: + - path: /books + method: PUT + - id: policy + path: /_plugins/_ism/policies/rollover + method: PUT + request: + payload: + policy: + description: A rollover policy. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 5 + status: [201] + output: + id: payload._id + - path: /_plugins/_ism/add/books + method: POST + request: + payload: + policy_id: ${policy.id} +chapters: + - synopsis: Change policy on an index. + path: /_plugins/_ism/change_policy/{index} + method: POST + parameters: + index: books + request: + payload: + policy_id: ${policy.id} + state: delete + include: + - state: searches + \ No newline at end of file diff --git a/tests/default/ism/explain.yaml b/tests/default/ism/explain.yaml new file mode 100644 index 000000000..51c71b9ec --- /dev/null +++ b/tests/default/ism/explain.yaml @@ -0,0 +1,40 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test getting the current state of the index. +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +prologues: + - path: /books + method: PUT + - id: policy + path: /_plugins/_ism/policies/rollover + method: PUT + request: + payload: + policy: + description: A rollover policy. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 5 + status: [201] + output: + id: payload._id + - path: /_plugins/_ism/add/books + method: POST + request: + payload: + policy_id: ${policy.id} +chapters: + - synopsis: Get the current policy for an index. + path: /_plugins/_ism/explain/{index} + method: GET + parameters: + index: books diff --git a/tests/default/ism/policies.yaml b/tests/default/ism/policies.yaml new file mode 100644 index 000000000..0a2f75e7f --- /dev/null +++ b/tests/default/ism/policies.yaml @@ -0,0 +1,81 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test index state management operations. +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create a rollover policy. + id: policy + path: /_plugins/_ism/policies/{policy_id} + method: PUT + parameters: + policy_id: rollover + request: + payload: + policy: + description: A rollover policy. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 5 + transitions: + - state_name: search + - name: search + actions: [] + transitions: + - state_name: delete + conditions: + min_index_age: 5m + - name: delete + actions: + - delete: {} + transitions: [] + ism_template: + index_patterns: ['log*'] + priority: 100 + error_notification: + destination: + chime: + url: https://example.com + message_template: + source: 'The index {{ctx.index}} failed during validation.' + response: + status: 201 + output: + primary_term: payload._primary_term + seq_no: payload._seq_no + - synopsis: Get a policy. + path: /_plugins/_ism/policies/{policy_id} + method: GET + parameters: + policy_id: rollover + - synopsis: Update a policy. + path: /_plugins/_ism/policies/{policy_id} + method: PUT + parameters: + policy_id: rollover + if_primary_term: ${policy.primary_term} + if_seq_no: ${policy.seq_no} + request: + payload: + policy: + description: Ingesting rollover. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 3 + response: + status: 200 + - synopsis: Delete a policy. + path: /_plugins/_ism/policies/{policy_id} + method: DELETE + parameters: + policy_id: rollover + response: + status: 200 diff --git a/tests/default/ism/remove.yaml b/tests/default/ism/remove.yaml new file mode 100644 index 000000000..f688cd519 --- /dev/null +++ b/tests/default/ism/remove.yaml @@ -0,0 +1,45 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test adding a policy to an index. +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +prologues: + - path: /books + method: PUT + - id: policy + path: /_plugins/_ism/policies/rollover + method: PUT + request: + payload: + policy: + description: A rollover policy. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 5 + status: [201] + output: + id: payload._id + - path: /_plugins/_ism/add/books + method: POST + request: + payload: + policy_id: ${policy.id} +chapters: + - synopsis: Remove the current policy from an index. + path: /_plugins/_ism/remove/{index} + method: POST + parameters: + index: books + - synopsis: Remove the current policy from an index that doesn't have one. + path: /_plugins/_ism/remove/{index} + method: POST + parameters: + index: books diff --git a/tests/default/transforms/preview.yaml b/tests/default/transforms/preview.yaml new file mode 100644 index 000000000..f26becd9f --- /dev/null +++ b/tests/default/transforms/preview.yaml @@ -0,0 +1,43 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test previewing a transform job's results. +prologues: + - path: /{index}/_doc + method: POST + parameters: + index: movies + refresh: true + request: + payload: + title: Beauty and the Beast + year: 91 + status: [201] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Preview a transform job's results. + path: /_plugins/_transform/_preview + method: POST + request: + payload: + transform: + enabled: false + description: A transform. + source_index: movies + target_index: films + data_selection_query: + match_all: {} + page_size: 10 + groups: + - terms: + source_field: year + target_field: produced + schedule: + interval: + period: 1 + unit: Minutes + start_time: 1602100553 + response: + status: 200 From de28026c568b436d10ffe0ea9794add0b85d9f06 Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Sun, 15 Sep 2024 22:36:52 -0700 Subject: [PATCH 18/73] remove knn from search request body (#580) * remove knn from search request body Signed-off-by: amberzsy * add CHANGELOG Signed-off-by: amberzsy --------- Signed-off-by: amberzsy --- CHANGELOG.md | 2 +- spec/namespaces/_core.yaml | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d91649a66..dc837e953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,7 +147,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) - Fixed query DSL `neural` field `query_image` set `contentEncoding` and `model_id` as optional ([#512](https://github.com/opensearch-project/opensearch-api-specification/pull/512)) -- Fixed `knn` query specification ([#538](https://github.com/opensearch-project/opensearch-api-specification/pull/538)) +- Fixed `knn` query specification ([#538](https://github.com/opensearch-project/opensearch-api-specification/pull/538))([#580](https://github.com/opensearch-project/opensearch-api-specification/pull/580)) - Fixed content-type for `/hot_threads` ([#543](https://github.com/opensearch-project/opensearch-api-specification/pull/543)) - Fixed `geo_distance` query spec ([#560](https://github.com/opensearch-project/opensearch-api-specification/pull/560)) - Fixed `/_cluster/settings` returning flat results ([#545](https://github.com/opensearch-project/opensearch-api-specification/pull/545)) diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index c9d5927be..3c2187e78 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2434,13 +2434,6 @@ components: type: array items: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/FieldAndFormat' - knn: - description: Defines the approximate kNN search to run. - oneOf: - - $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/KnnQuery' - - type: array - items: - $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/KnnQuery' rank: $ref: '../schemas/_common.yaml#/components/schemas/RankContainer' min_score: From f167e0d42395b920c389a7799fe97aff2b9b447d Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Sun, 15 Sep 2024 22:39:35 -0700 Subject: [PATCH 19/73] =?UTF-8?q?remove=20extra=20free-format=20object=20i?= =?UTF-8?q?n=20create/index/delete=20operation=20from=E2=80=A6=20(#582)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove extra free-format object in create/index/delete operation from bulk api Signed-off-by: amberzsy * add CHANGELOG Signed-off-by: amberzsy --------- Signed-off-by: amberzsy --- CHANGELOG.md | 1 + spec/schemas/_core.bulk.yaml | 12 +++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc837e953..9886bd69b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,6 +156,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) +- Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) ### Security diff --git a/spec/schemas/_core.bulk.yaml b/spec/schemas/_core.bulk.yaml index 9006cfe23..417d1ffd4 100644 --- a/spec/schemas/_core.bulk.yaml +++ b/spec/schemas/_core.bulk.yaml @@ -20,9 +20,7 @@ components: minProperties: 1 maxProperties: 1 IndexOperation: - allOf: - - $ref: '#/components/schemas/WriteOperation' - - type: object + $ref: '#/components/schemas/WriteOperation' WriteOperation: allOf: - $ref: '#/components/schemas/OperationBase' @@ -64,9 +62,7 @@ components: version_type: $ref: '_common.yaml#/components/schemas/VersionType' CreateOperation: - allOf: - - $ref: '#/components/schemas/WriteOperation' - - type: object + $ref: '#/components/schemas/WriteOperation' UpdateOperation: allOf: - $ref: '#/components/schemas/OperationBase' @@ -78,9 +74,7 @@ components: retry_on_conflict: type: number DeleteOperation: - allOf: - - $ref: '#/components/schemas/OperationBase' - - type: object + $ref: '#/components/schemas/OperationBase' UpdateAction: type: object properties: From 811a5c9590e692252dfba9cb0da8039dcfbcb82b Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Mon, 16 Sep 2024 08:54:00 -0700 Subject: [PATCH 20/73] fix issues for termsQuery (#570) --- CHANGELOG.md | 2 +- spec/schemas/_common.query_dsl.yaml | 53 +++++----- .../{terms.yaml => terms/array_terms.yaml} | 6 +- .../query/terms/array_terms_with_boost.yaml | 62 ++++++++++++ .../search/query/terms/terms_lookup.yaml | 99 +++++++++++++++++++ 5 files changed, 196 insertions(+), 26 deletions(-) rename tests/default/_core/search/query/{terms.yaml => terms/array_terms.yaml} (96%) create mode 100644 tests/default/_core/search/query/terms/array_terms_with_boost.yaml create mode 100644 tests/default/_core/search/query/terms/terms_lookup.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9886bd69b..3687ac3d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `AwarenessAttributeStats` to `/_cluster/health` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cache_reserved_in_bytes` to `ClusterFileSystem` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) - Added `cluster_manager` to `ClusterNodeCount` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534)) -- Added support for `query` with `terms` in `_search` ([#546](https://github.com/opensearch-project/opensearch-api-specification/pull/546))([#564](https://github.com/opensearch-project/opensearch-api-specification/pull/564)) +- Added support for `query` with `terms` in `_search` ([#546](https://github.com/opensearch-project/opensearch-api-specification/pull/546))([#564](https://github.com/opensearch-project/opensearch-api-specification/pull/564))([#570](https://github.com/opensearch-project/opensearch-api-specification/pull/570)) - Added `h`, `bytes`, `time` parameters to cat API tests ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Added `terminated_early` to `count@200` ([#547](https://github.com/opensearch-project/opensearch-api-specification/pull/547)) - Added request and response schemas for `/_cluster/routing/awareness/{attribute}/weights` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index c98fa14f5..2a23d5ccf 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -190,15 +190,9 @@ components: minProperties: 1 maxProperties: 1 terms: - type: object - additionalProperties: - anyOf: - - $ref: '#/components/schemas/TermsQuery' - - type: array - items: - type: string - minProperties: 1 - maxProperties: 1 + description: |- + Returns documents that contain one or more exact terms in a provided field. + $ref: '#/components/schemas/TermsQueryField' terms_set: description: |- Returns documents that contain a minimum number of exact terms in a provided field. @@ -1877,19 +1871,34 @@ components: type: boolean required: - value - TermsQuery: - anyOf: - - $ref: '#/components/schemas/QueryBase' - - type: object - properties: - index: - $ref: '_common.yaml#/components/schemas/IndexName' - id: - $ref: '_common.yaml#/components/schemas/Id' - path: - $ref: '_common.yaml#/components/schemas/Field' - routing: - $ref: '_common.yaml#/components/schemas/Routing' + TermsQueryField: + type: object + properties: + boost: + description: + |- + A floating-point value that specifies the weight of this field toward the relevance score. + Values above 1.0 increase the field’s relevance. Values between 0.0 and 1.0 decrease the field’s relevance. + Default is 1.0. + type: number + format: float + additionalProperties: + oneOf: + - $ref: '#/components/schemas/TermsLookupField' + - type: array + items: + type: string + TermsLookupField: + type: object + properties: + index: + $ref: '_common.yaml#/components/schemas/IndexName' + id: + $ref: '_common.yaml#/components/schemas/Id' + path: + $ref: '_common.yaml#/components/schemas/Field' + routing: + $ref: '_common.yaml#/components/schemas/Routing' TermsSetQuery: allOf: - $ref: '#/components/schemas/QueryBase' diff --git a/tests/default/_core/search/query/terms.yaml b/tests/default/_core/search/query/terms/array_terms.yaml similarity index 96% rename from tests/default/_core/search/query/terms.yaml rename to tests/default/_core/search/query/terms/array_terms.yaml index b48375901..13b2020f4 100644 --- a/tests/default/_core/search/query/terms.yaml +++ b/tests/default/_core/search/query/terms/array_terms.yaml @@ -1,6 +1,6 @@ -$schema: ../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../json_schemas/test_story.schema.yaml -description: Comprehensive test suite for TermsQuery, including array of terms and term lookup. +description: Comprehensive test suite for TermsQuery, including array of terms. version: '>= 1.2' prologues: @@ -143,4 +143,4 @@ chapters: _score: 1 _source: title: Cyberpunk 2077 - genre: RPG + genre: RPG \ No newline at end of file diff --git a/tests/default/_core/search/query/terms/array_terms_with_boost.yaml b/tests/default/_core/search/query/terms/array_terms_with_boost.yaml new file mode 100644 index 000000000..5b7141117 --- /dev/null +++ b/tests/default/_core/search/query/terms/array_terms_with_boost.yaml @@ -0,0 +1,62 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Comprehensive test suite for TermsQuery, including array of terms with boost top level parameter. +version: '>= 1.2' + +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + title: + type: text + genre: + type: keyword + director_id: + type: keyword + status: [200] + + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: Beauty and the Beast + genre: adventure + status: [201] + +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Search using TermsQuery with an array of terms with boost. + path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + query: + terms: + boost: 1.6 + genre: + - adventure + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + hits: + - _index: movies + _score: 1.6 + _source: + title: Beauty and the Beast + genre: adventure \ No newline at end of file diff --git a/tests/default/_core/search/query/terms/terms_lookup.yaml b/tests/default/_core/search/query/terms/terms_lookup.yaml new file mode 100644 index 000000000..3b6b6cf30 --- /dev/null +++ b/tests/default/_core/search/query/terms/terms_lookup.yaml @@ -0,0 +1,99 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Comprehensive test suite for TermsQuery, including term lookup. +version: '>= 1.2' + +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + title: + type: text + genre: + type: keyword + director_id: + type: keyword + status: [200] + + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: The Lion King + genre: animation + status: [201] + + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + title: Beauty and the Beast + genre: adventure + status: [201] + + - path: /theater + method: PUT + request: + payload: + mappings: + properties: + name: + type: text + genre: + type: keyword + status: [200] + + - path: /theater/_doc/101 + method: POST + parameters: + refresh: true + request: + payload: + name: AMC + genre: [adventure] + status: [201] + +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + + - path: /theater + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Search using terms lookup. + path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + query: + terms: + genre: + index: theater + id: '101' + path: genre + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: eq + hits: + - _index: movies + _score: 1 + _source: + title: Beauty and the Beast + genre: adventure \ No newline at end of file From 8fe6a49881224b594ba19727a365fff8357d9365 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 16 Sep 2024 21:42:52 +0200 Subject: [PATCH 21/73] Ignore superseded operations in coverage reports. (#575) --- tools/src/tester/TestResults.ts | 11 +++++++---- tools/src/tester/types/test.types.ts | 4 ++-- tools/tests/tester/MergedOpenApiSpec.test.ts | 1 + .../specs/complete/_superseded_operations.yaml | 7 ++++++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tools/src/tester/TestResults.ts b/tools/src/tester/TestResults.ts index 864ea05dc..3b79d2ad2 100644 --- a/tools/src/tester/TestResults.ts +++ b/tools/src/tester/TestResults.ts @@ -47,11 +47,14 @@ export default class TestResults { operations(): Operation[] { if (this._operations !== undefined) return this._operations - this._operations = _.uniqWith(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => { - return Object.values(path_item).map((method) => { - return { method: method.toUpperCase(), path } + + this._operations = _.uniqWith(_.compact(Object.entries(this._spec.spec().paths).flatMap(([path, ops]) => { + return Object.entries(ops as Record).map(([method, spec]) => { + if (spec['x-ignorable'] !== true) { + return { method: method.toUpperCase(), path } + } }) - }), isEqual) + })), isEqual) return this._operations } diff --git a/tools/src/tester/types/test.types.ts b/tools/src/tester/types/test.types.ts index f98c93e9b..15fc0cabb 100644 --- a/tools/src/tester/types/test.types.ts +++ b/tools/src/tester/types/test.types.ts @@ -12,9 +12,9 @@ import { Operation } from "./eval.types" export interface SpecTestCoverage { summary: { total_operations_count: number - evaluated_operations_count: number, + evaluated_operations_count: number evaluated_paths_pct: number }, - operations: Operation[], + operations: Operation[] evaluated_operations: Operation[] } diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index db852a030..3857cae52 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -22,6 +22,7 @@ describe('merged API spec', () => { test('paths', () => { expect(spec.paths()).toEqual({ '/_nodes/{id}': ['get', 'post'], + '/_superseded/nodes/{id}': ['get'], '/cluster_manager': ['get', 'post'], '/index': ['get'], '/nodes': ['get'] diff --git a/tools/tests/tester/fixtures/specs/complete/_superseded_operations.yaml b/tools/tests/tester/fixtures/specs/complete/_superseded_operations.yaml index a5d5e9940..ff4c5590c 100644 --- a/tools/tests/tester/fixtures/specs/complete/_superseded_operations.yaml +++ b/tools/tests/tester/fixtures/specs/complete/_superseded_operations.yaml @@ -1 +1,6 @@ -$schema: should-be-ignored \ No newline at end of file +$schema: should-be-ignored + +/_superseded/nodes/{id}: + superseded_by: /_nodes/{id} + operations: + - GET From d4d4fa9eefe3401ed1dba64b64fd0053de18e4de Mon Sep 17 00:00:00 2001 From: Junwei Dai <59641585+junweid62@users.noreply.github.com> Date: Tue, 17 Sep 2024 00:21:46 -0700 Subject: [PATCH 22/73] Flow_framework spec refactor: Added Tests, Param, and Error Types (#574) * refactor: 1. Added missing param for update workflow request body 2. Added test for create workflow with sample test Signed-off-by: Junwei Dai * Update schema to use application/json content type Signed-off-by: Junwei Dai * Update schema to use application/json content type Signed-off-by: Junwei Dai * Update the template link to pass the link check Signed-off-by: Junwei Dai --------- Signed-off-by: Junwei Dai Co-authored-by: Junwei Dai --- CHANGELOG.md | 1 + spec/namespaces/flow_framework.yaml | 7 + spec/schemas/flow_framework.common.yaml | 2 + spec/schemas/flow_framework.errors.yaml | 53 +++- tests/default/flow_framework/deprovision.yaml | 20 -- tests/default/flow_framework/provision.yaml | 22 +- tests/default/flow_framework/template.yaml | 237 ++++++++++++++++++ tests/default/flow_framework/workflow.yaml | 10 - 8 files changed, 291 insertions(+), 61 deletions(-) create mode 100644 tests/default/flow_framework/template.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 3687ac3d3..23e37961c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -94,6 +94,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Added `/_plugins/_flow_framework/` ,`_search`,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) +- Added more 4xx errors to `/_plugins/_flow_framework/` and added sample template test ([#833](https://github.com/opensearch-project/flow-framework/issues/833) ### Changed diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index e3a3e005c..feed463e6 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -412,6 +412,10 @@ components: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNameRequiredError' flow_framework.update@400: description: Bad Request - Multiple possible reasons content: @@ -423,6 +427,9 @@ components: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' flow_framework.update@201: content: application/json: diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index a8d7d7c00..d3ab4ae9b 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -65,6 +65,8 @@ components: use_case: type: string description: A use case, which can be used with the Search Workflow API to find related workflows. + workflows: + type: object version: $ref: '#/components/schemas/version' minProperties: 1 diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 6f46f2435..879cf7fe5 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -11,7 +11,7 @@ components: application/json: type: object properties: - message: + error: type: string example: This API is disabled. To enable it, set [flow_framework.enabled] to true. status: @@ -21,7 +21,7 @@ components: application/json: type: object properties: - message: + error: type: string example: Only the parameters [param1, param2] are permitted unless the provision parameter is set to true. status: @@ -31,7 +31,7 @@ components: application/json: type: object properties: - message: + error: type: string example: You cannot use both the 'provision_workflow' and 'update_workflow_fields' parameters in the same request. status: @@ -41,7 +41,7 @@ components: application/json: type: object properties: - message: + error: type: string example: You cannot use the 'reprovision_workflow' parameter to create a new template. status: @@ -51,7 +51,7 @@ components: application/json: type: object properties: - message: + error: type: string example: You cannot use the 'reprovision_workflow' and 'use_case' parameters in the same request. status: @@ -61,7 +61,7 @@ components: application/json: type: object properties: - message: + error: type: string example: Workflow ID can not be null status: @@ -100,7 +100,7 @@ components: application/json: type: object properties: - message: + error: type: string example: Failed to save workflow state status: @@ -110,27 +110,60 @@ components: application/json: type: object properties: - message: + error: type: string example: Maximum workflows limit reached 50 code: type: integer + TemplateNameRequiredError: + type: object + properties: + error: + type: string + description: Error message when the template name is missing. TemplateNotFoundError: content: application/json: type: object properties: - message: + error: type: string example: Failed to retrieve template (12345) from global context. code: type: integer + InvalidTemplateVersionError: + content: + application/json: + type: object + properties: + error: + type: string + description: Error message when the template version is invalid or missing. + example: Unable to parse field [version] in a version object. + UnsupportedFieldUpdateError: + content: + application/json: + type: object + properties: + error: + type: string + description: Error message when trying to update an unsupported field in a template. + example: You can not update the field [fieldName] without updating the whole template. + WorkflowParsingError: + content: + application/json: + type: object + properties: + error: + type: string + description: Error message when workflow parsing fails. + example: Unable to parse field [workflow] in a template object. WorkflowStepsRetrieveError: content: application/json: type: object properties: - message: + error: type: string example: Failed to retrieve workflow step json. code: diff --git a/tests/default/flow_framework/deprovision.yaml b/tests/default/flow_framework/deprovision.yaml index 43babeeb3..dada6f1f2 100644 --- a/tests/default/flow_framework/deprovision.yaml +++ b/tests/default/flow_framework/deprovision.yaml @@ -25,16 +25,6 @@ epilogues: parameters: workflow_id: ${create_flow_framework.test_workflow_id} chapters: - - synopsis: Check the provision status before calling the deprovision API. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - payload: - workflow_id: ${create_flow_framework.test_workflow_id} - state: PROVISIONING - synopsis: Deprovision workflow. path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision method: POST @@ -42,16 +32,6 @@ chapters: workflow_id: ${create_flow_framework.test_workflow_id} response: status: 200 - - synopsis: Check the provision status after calling the deprovision API. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - payload: - workflow_id: ${create_flow_framework.test_workflow_id} - state: NOT_STARTED - synopsis: Deprovision workflow using an invalid ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision method: POST diff --git a/tests/default/flow_framework/provision.yaml b/tests/default/flow_framework/provision.yaml index 2d52e83e1..fb1e41a0d 100644 --- a/tests/default/flow_framework/provision.yaml +++ b/tests/default/flow_framework/provision.yaml @@ -23,16 +23,6 @@ epilogues: parameters: workflow_id: ${create_flow_framework.test_workflow_id} chapters: - - synopsis: Check the status before calling provision API. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - payload: - workflow_id: ${create_flow_framework.test_workflow_id} - state: NOT_STARTED - synopsis: Provision workflow. path: /_plugins/_flow_framework/workflow/{workflow_id}/_provision method: POST @@ -42,14 +32,4 @@ chapters: payload: openai_key: '1234556' response: - status: 200 - - synopsis: Check the provision status after calling provision API. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - payload: - workflow_id: ${create_flow_framework.test_workflow_id} - state: PROVISIONING \ No newline at end of file + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/template.yaml b/tests/default/flow_framework/template.yaml new file mode 100644 index 000000000..c7ac780c9 --- /dev/null +++ b/tests/default/flow_framework/template.yaml @@ -0,0 +1,237 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework endpoints using sample template. +version: '>= 2.12' +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Create workflow with openAI model. + id: create_flow_framework + path: /_plugins/_flow_framework/workflow + method: POST + request: + payload: + name: Deploy OpenAI Model + description: Deploy a model using a connector to OpenAI + use_case: PROVISION + version: + template: 1.0.0 + compatibility: + - 2.12.0 + - 3.0.0 + workflows: + provision: + nodes: + - id: create_openai_connector + type: create_connector + user_inputs: + name: OpenAI Chat Connector + description: The connector to public OpenAI model service for GPT 3.5 + version: '1' + protocol: http + parameters: + endpoint: api.openai.com + model: gpt-3.5-turbo + response_filter: '$.choices[0].message.content' + credential: + openAI_key: '1234556' + actions: + - action_type: predict + method: POST + url: example_url + - id: register_openai_model + type: register_remote_model + previous_node_inputs: + create_openai_connector: connector_id + user_inputs: + name: openAI-gpt-3.5-turbo + - id: deploy_openai_model + type: deploy_model + previous_node_inputs: + register_openai_model: model_id + response: + status: 201 + output: + test_workflow_id: payload.workflow_id + - synopsis: Update workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + update_fields: true + request: + payload: + name: test_update_name + response: + status: 201 + - synopsis: Update workflow field fail without updating the whole template. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + update_fields: true + request: + payload: + workflows: + name: test + response: + status: 400 + - synopsis: Update workflow with updating the whole template. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + request: + payload: + name: Deploy OpenAI Model + description: Deploy a model using a connector to OpenAI + use_case: PROVISION + version: + template: 1.0.0 + compatibility: + - 2.12.0 + - 3.0.0 + workflows: + provision: + nodes: + - id: create_openai_connector + type: create_connector + user_inputs: + name: OpenAI Chat Connector + description: The connector to public OpenAI model service for GPT 3.5 update + version: '1' + protocol: http + parameters: + endpoint: api.openai.com + model: gpt-3.5-turbo + response_filter: '$.choices[0].message.content' + credential: + openAI_key: '1234556' + actions: + - action_type: predict + method: POST + url: example_url + - id: register_openai_model + type: register_remote_model + previous_node_inputs: + create_openai_connector: connector_id + user_inputs: + name: openAI-gpt-3.5-turbo + - id: deploy_openai_model + type: deploy_model + previous_node_inputs: + register_openai_model: model_id + response: + status: 201 + - synopsis: Provision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_provision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Update workflow fail With provision and update_fields set true. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + provision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow fail With Reprovision and updateFields set true. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow fail With Reprovision set true. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + reprovision: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow fail With Reprovision set true and UseCase Not null. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow fail With Reprovision set true and UseCase Not null. + version: '>= 2.17' + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Get workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Get workflow fail. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: GET + parameters: + workflow_id: invalid + response: + status: 404 + - synopsis: Update workflow with a non-existent workflow ID. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: invalid + request: + payload: + name: test_update_work_flow + response: + status: 404 + - synopsis: Search workflow state. + path: /_plugins/_flow_framework/workflow/state/_search + method: POST + request: + payload: + query: + match: + state: PROVISIONING + response: + status: 200 + - synopsis: Delete workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index d136c8047..4f50a012d 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -118,16 +118,6 @@ chapters: name: test_update_work_flow response: status: 404 - - synopsis: Search workflow state. - path: /_plugins/_flow_framework/workflow/state/_search - method: POST - request: - payload: - query: - match: - state: test - response: - status: 200 - synopsis: Delete workflow. path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE From 3d49360c45c33f4aebf09a50ef9e06a27a225e66 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Sep 2024 16:47:31 +0200 Subject: [PATCH 23/73] Completed ISM specs and tests. (#578) --- CHANGELOG.md | 6 +- spec/_superseded_operations.yaml | 2 + spec/namespaces/ism.yaml | 309 +++++++++++++++++- spec/schemas/ism._common.yaml | 49 ++- tests/default/ism/add.yaml | 25 +- tests/default/ism/change_policy.yaml | 28 +- tests/default/ism/explain.yaml | 33 +- tests/default/ism/policies.yaml | 49 ++- .../default/ism/refresh_search_analyzers.yaml | 43 +++ tests/default/ism/remove.yaml | 23 +- tests/default/ism/retry.yaml | 43 +++ 11 files changed, 548 insertions(+), 62 deletions(-) create mode 100644 tests/default/ism/refresh_search_analyzers.yaml create mode 100644 tests/default/ism/retry.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 23e37961c..0129e440f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,9 +92,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) -- Added `/_plugins/_flow_framework/` ,`_search`,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) -- Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) -- Added more 4xx errors to `/_plugins/_flow_framework/` and added sample template test ([#833](https://github.com/opensearch-project/flow-framework/issues/833) +- Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833)) +- Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) +- Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) ### Changed diff --git a/spec/_superseded_operations.yaml b/spec/_superseded_operations.yaml index e4170cc53..80e556067 100644 --- a/spec/_superseded_operations.yaml +++ b/spec/_superseded_operations.yaml @@ -189,10 +189,12 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml superseded_by: /_plugins/_ism/explain operations: - GET + - POST /_opendistro/_ism/explain/{index}: superseded_by: /_plugins/_ism/explain/{index} operations: - GET + - POST /_opendistro/_ism/policies: superseded_by: /_plugins/_ism/policies operations: diff --git a/spec/namespaces/ism.yaml b/spec/namespaces/ism.yaml index 3aa5688f4..334bc69f7 100644 --- a/spec/namespaces/ism.yaml +++ b/spec/namespaces/ism.yaml @@ -4,18 +4,56 @@ info: description: Use the index state management operations to programmatically work with policies and managed indexes. version: 1.0.0 paths: + /_plugins/_ism/policies: + get: + operationId: ism.get_policies.0 + x-operation-group: ism.get_policies + description: Gets the policies. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#get-policy + responses: + '200': + $ref: '#/components/responses/ism.get_policies@200' + put: + operationId: ism.put_policies.0 + x-operation-group: ism.put_policies + description: Gets the policies. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#create-policy + requestBody: + $ref: '#/components/requestBodies/ism.put_policies' + parameters: + - $ref: '#/components/parameters/ism.put_policies::query.if_primary_term' + - $ref: '#/components/parameters/ism.put_policies::query.if_seq_no' + - $ref: '#/components/parameters/ism.put_policies::query.policyID' + responses: + '200': + $ref: '#/components/responses/ism.put_policies@200' /_plugins/_ism/policies/{policy_id}: get: operationId: ism.get_policy.0 x-operation-group: ism.get_policy - description: Gets the policy. + description: Gets a policy. externalDocs: - url: https://opensearch.org/docs/latest/im-plugin/ism/api/#get-policy + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#put-policy parameters: - $ref: '#/components/parameters/ism.get_policy::path.policy_id' responses: '200': $ref: '#/components/responses/ism.get_policy@200' + head: + operationId: ism.exists_policy.0 + x-operation-group: ism.exists_policy + description: Checks whether the policy exists. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#get-policy + parameters: + - $ref: '#/components/parameters/ism.exists_policy::path.policy_id' + responses: + '200': + $ref: '#/components/responses/ism.exists_policy@200' + '404': + $ref: '#/components/responses/ism.exists_policy@404' put: operationId: ism.put_policy.0 x-operation-group: ism.put_policy @@ -44,60 +82,187 @@ paths: responses: '200': $ref: '#/components/responses/ism.delete_policy@200' - /_plugins/_ism/add/{index}: + /_plugins/_ism/add: post: operationId: ism.add_policy.0 x-operation-group: ism.add_policy description: Adds a policy to an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#add-policy + parameters: + - $ref: '#/components/parameters/ism.add_policy::query.index' + requestBody: + $ref: '#/components/requestBodies/ism.add_policy' + responses: + '200': + $ref: '#/components/responses/ism.add_policy@200' + /_plugins/_ism/add/{index}: + post: + operationId: ism.add_policy.1 + x-operation-group: ism.add_policy + description: Adds a policy to an index. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/ism/api/#add-policy parameters: - $ref: '#/components/parameters/ism.add_policy::path.index' + - $ref: '#/components/parameters/ism.add_policy::query.index' requestBody: $ref: '#/components/requestBodies/ism.add_policy' responses: '200': $ref: '#/components/responses/ism.add_policy@200' - /_plugins/_ism/remove/{index}: + /_plugins/_ism/remove: post: operationId: ism.remove_policy.0 x-operation-group: ism.remove_policy description: Removes a policy from an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#remove-policy + parameters: + - $ref: '#/components/parameters/ism.remove_policy::query.index' + responses: + '200': + $ref: '#/components/responses/ism.remove_policy@200' + /_plugins/_ism/remove/{index}: + post: + operationId: ism.remove_policy.1 + x-operation-group: ism.remove_policy + description: Removes a policy from an index. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/ism/api/#remove-policy parameters: - $ref: '#/components/parameters/ism.remove_policy::path.index' + - $ref: '#/components/parameters/ism.remove_policy::query.index' responses: '200': $ref: '#/components/responses/ism.remove_policy@200' - /_plugins/_ism/change_policy/{index}: + /_plugins/_ism/change_policy: post: operationId: ism.change_policy.0 x-operation-group: ism.change_policy description: Updates the managed index policy to a new policy. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#update-managed-index-policy + parameters: + - $ref: '#/components/parameters/ism.change_policy::query.index' + requestBody: + $ref: '#/components/requestBodies/ism.change_policy' + responses: + '200': + $ref: '#/components/responses/ism.change_policy@200' + /_plugins/_ism/change_policy/{index}: + post: + operationId: ism.change_policy.1 + x-operation-group: ism.change_policy + description: Updates the managed index policy to a new policy. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/ism/api/#update-managed-index-policy parameters: - $ref: '#/components/parameters/ism.change_policy::path.index' + - $ref: '#/components/parameters/ism.change_policy::query.index' requestBody: $ref: '#/components/requestBodies/ism.change_policy' responses: '200': $ref: '#/components/responses/ism.change_policy@200' + /_plugins/_ism/explain: + get: + operationId: ism.explain_policy.0 + x-operation-group: ism.explain_policy + description: Gets the currently applied policy on indices. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#explain-index + requestBody: + $ref: '#/components/requestBodies/ism.explain_policy' + responses: + '200': + $ref: '#/components/responses/ism.explain_policy@200' + post: + operationId: ism.explain_policy.0 + x-operation-group: ism.explain_policy + description: Gets the currently applied policy on indices. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#explain-index + requestBody: + $ref: '#/components/requestBodies/ism.explain_policy' + responses: + '200': + $ref: '#/components/responses/ism.explain_policy@200' /_plugins/_ism/explain/{index}: get: - operationId: ism.explain_index.0 - x-operation-group: ism.explain_index - description: Gets the currently applied policy on an index. + operationId: ism.explain_policy.1 + x-operation-group: ism.explain_policy + description: Gets the currently applied policy on indices. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/ism/api/#explain-index parameters: - - $ref: '#/components/parameters/ism.explain_index::path.index' + - $ref: '#/components/parameters/ism.explain_policy::path.index' + requestBody: + $ref: '#/components/requestBodies/ism.explain_policy' responses: '200': - $ref: '#/components/responses/ism.explain_index@200' + $ref: '#/components/responses/ism.explain_policy@200' + post: + operationId: ism.explain_policy.1 + x-operation-group: ism.explain_policy + description: Gets the currently applied policy on indices. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#explain-index + parameters: + - $ref: '#/components/parameters/ism.explain_policy::path.index' + requestBody: + $ref: '#/components/requestBodies/ism.explain_policy' + responses: + '200': + $ref: '#/components/responses/ism.explain_policy@200' + /_plugins/_ism/retry: + post: + operationId: ism.retry_index.0 + x-operation-group: ism.retry_index + description: Retry the failed action for an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#retry-failed-index + parameters: + - $ref: '#/components/parameters/ism.retry_index::query.index' + requestBody: + $ref: '#/components/requestBodies/ism.retry_index' + responses: + '200': + $ref: '#/components/responses/ism.retry_index@200' + /_plugins/_ism/retry/{index}: + post: + operationId: ism.retry_index.1 + x-operation-group: ism.retry_index + description: Retry the failed action for an index. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/ism/api/#retry-failed-index + parameters: + - $ref: '#/components/parameters/ism.retry_index::path.index' + - $ref: '#/components/parameters/ism.retry_index::query.index' + requestBody: + $ref: '#/components/requestBodies/ism.retry_index' + responses: + '200': + $ref: '#/components/responses/ism.retry_index@200' + /_plugins/_refresh_search_analyzers/{index}: + post: + operationId: ism.refresh_search_analyzers.0 + x-operation-group: ism.refresh_search_analyzers + description: Refresh search analyzers in real time. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/refresh-analyzer/ + parameters: + - $ref: '#/components/parameters/ism.refresh_search_analyzers::path.index' + responses: + '200': + $ref: '#/components/responses/ism.refresh_search_analyzers@200' components: requestBodies: + ism.put_policies: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyRequest' ism.put_policy: content: application/json: @@ -113,7 +278,29 @@ components: application/json: schema: $ref: '../schemas/ism._common.yaml#/components/schemas/ChangePolicyRequest' + ism.retry_index: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/RetryIndexRequest' + ism.explain_policy: + content: + application/json: + schema: {} responses: + ism.exists_policy@200: + content: + application/json: + schema: {} + ism.exists_policy@404: + content: + application/json: + schema: {} + ism.get_policies@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/GetPoliciesResponse' ism.get_policy@200: content: application/json: @@ -124,6 +311,11 @@ components: application/json: schema: $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyResponse' + ism.put_policies@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/PutPolicyResponse' ism.put_policy@200: content: application/json: @@ -149,12 +341,28 @@ components: application/json: schema: $ref: '../schemas/ism._common.yaml#/components/schemas/ChangePolicyResponse' - ism.explain_index@200: + ism.explain_policy@200: content: application/json: schema: $ref: '../schemas/ism._common.yaml#/components/schemas/ExplainIndexResponse' + ism.retry_index@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/RetryIndexResponse' + ism.refresh_search_analyzers@200: + content: + application/json: + schema: + $ref: '../schemas/ism._common.yaml#/components/schemas/RefreshSearchAnalyzersResponse' parameters: + ism.exists_policy::path.policy_id: + name: policy_id + in: path + required: true + schema: + type: string ism.get_policy::path.policy_id: name: policy_id in: path @@ -187,6 +395,15 @@ components: schema: type: number style: form + ism.add_policy::query.index: + name: index + in: query + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' ism.add_policy::path.index: name: index in: path @@ -197,6 +414,24 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple + ism.change_policy::query.index: + name: index + in: query + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + ism.remove_policy::query.index: + name: index + in: query + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' ism.remove_policy::path.index: name: index in: path @@ -217,7 +452,26 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - ism.explain_index::path.index: + ism.explain_policy::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + ism.retry_index::query.index: + name: index + in: query + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + ism.retry_index::path.index: name: index in: path description: |- @@ -227,3 +481,34 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple + ism.refresh_search_analyzers::path.index: + name: index + in: path + description: |- + Comma-separated list of data streams, indices, and aliases. + Supports wildcards (`*`). + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + ism.put_policies::query.policyID: + name: policyID + in: query + required: true + schema: + type: string + style: form + ism.put_policies::query.if_primary_term: + name: if_primary_term + in: query + description: Only perform the operation if the document has this primary term. + schema: + type: number + style: form + ism.put_policies::query.if_seq_no: + name: if_seq_no + in: query + description: Only perform the operation if the document has this sequence number. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/SequenceNumber' + style: form diff --git a/spec/schemas/ism._common.yaml b/spec/schemas/ism._common.yaml index 19e0762ce..73cb42071 100644 --- a/spec/schemas/ism._common.yaml +++ b/spec/schemas/ism._common.yaml @@ -49,6 +49,11 @@ components: - type: array items: $ref: '#/components/schemas/IsmTemplate' + PolicyWithMetadata: + type: object + allOf: + - $ref: '#/components/schemas/Metadata' + - $ref: '#/components/schemas/PolicyEnvelope' States: type: object description: |- @@ -217,10 +222,17 @@ components: conditions: type: object additionalProperties: true + GetPoliciesResponse: + type: object + properties: + total_policies: + type: number + policies: + type: array + items: + $ref: '#/components/schemas/PolicyWithMetadata' GetPolicyResponse: - allOf: - - $ref: '#/components/schemas/Metadata' - - $ref: '#/components/schemas/PolicyEnvelope' + $ref: '#/components/schemas/PolicyWithMetadata' PutPolicyResponse: allOf: - $ref: '#/components/schemas/Metadata' @@ -230,7 +242,7 @@ components: $ref: '#/components/schemas/PolicyEnvelope' DeletePolicyResponse: $ref: '_common.yaml#/components/schemas/WriteResponseBase' - ChangePolicyResponse: + ChangeResponse: type: object properties: updated_indices: @@ -241,6 +253,8 @@ components: type: array items: $ref: '#/components/schemas/FailedIndex' + ChangePolicyResponse: + $ref: '#/components/schemas/ChangeResponse' FailedIndex: type: object properties: @@ -374,3 +388,30 @@ components: type: number last_updated_time: type: integer + RetryIndexRequest: + type: object + properties: + state: + type: string + required: + - state + RetryIndexResponse: + $ref: '#/components/schemas/ChangeResponse' + RefreshSearchAnalyzersResponse: + type: object + properties: + _shards: + $ref: '_common.yaml#/components/schemas/ShardStatistics' + successful_refresh_details: + type: array + items: + $ref: '#/components/schemas/RefreshSearchAnalyzersResponseDetails' + RefreshSearchAnalyzersResponseDetails: + type: object + properties: + index: + type: string + refreshed_analyzers: + type: array + items: + type: string diff --git a/tests/default/ism/add.yaml b/tests/default/ism/add.yaml index f5283aa96..53794f33c 100644 --- a/tests/default/ism/add.yaml +++ b/tests/default/ism/add.yaml @@ -1,13 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test adding a policy to an index. -epilogues: - - path: /_plugins/_ism/policies/rollover - method: DELETE - status: [200, 404] - - path: /books - method: DELETE - status: [200, 404] prologues: - path: /books method: PUT @@ -28,7 +21,7 @@ prologues: output: id: payload._id chapters: - - synopsis: Add a rollover policy to the books index. + - synopsis: Add a rollover policy to the books index with the index in path. path: /_plugins/_ism/add/{index} method: POST parameters: @@ -36,4 +29,18 @@ chapters: request: payload: policy_id: ${policy.id} - \ No newline at end of file + - synopsis: Add a rollover policy to the books index with the index in a query parameter. + path: /_plugins/_ism/add + method: POST + parameters: + index: books + request: + payload: + policy_id: ${policy.id} +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/change_policy.yaml b/tests/default/ism/change_policy.yaml index 3268f0d0a..cc7af8514 100644 --- a/tests/default/ism/change_policy.yaml +++ b/tests/default/ism/change_policy.yaml @@ -1,13 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test updating a managed index policy. -epilogues: - - path: /_plugins/_ism/policies/rollover - method: DELETE - status: [200, 404] - - path: /books - method: DELETE - status: [200, 404] prologues: - path: /books method: PUT @@ -33,7 +26,18 @@ prologues: payload: policy_id: ${policy.id} chapters: - - synopsis: Change policy on an index. + - synopsis: Change policy on an index in the query parameter. + path: /_plugins/_ism/change_policy + method: POST + parameters: + index: books + request: + payload: + policy_id: ${policy.id} + state: delete + include: + - state: searches + - synopsis: Change policy on an index in the path. path: /_plugins/_ism/change_policy/{index} method: POST parameters: @@ -44,4 +48,10 @@ chapters: state: delete include: - state: searches - \ No newline at end of file +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/explain.yaml b/tests/default/ism/explain.yaml index 51c71b9ec..f3cf7654d 100644 --- a/tests/default/ism/explain.yaml +++ b/tests/default/ism/explain.yaml @@ -1,13 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test getting the current state of the index. -epilogues: - - path: /_plugins/_ism/policies/rollover - method: DELETE - status: [200, 404] - - path: /books - method: DELETE - status: [200, 404] prologues: - path: /books method: PUT @@ -33,8 +26,32 @@ prologues: payload: policy_id: ${policy.id} chapters: - - synopsis: Get the current policy for an index. + - synopsis: Get the total managed indices count (GET). + path: /_plugins/_ism/explain + method: GET + - synopsis: Get the total managed indices count (POST). + version: '>= 2.12' + path: /_plugins/_ism/explain + method: POST + request: + payload: {} + - synopsis: Get the current policy for an index (GET). path: /_plugins/_ism/explain/{index} method: GET parameters: index: books + - synopsis: Get the current policy for an index (POST). + version: '>= 2.12' + path: /_plugins/_ism/explain/{index} + method: POST + parameters: + index: books + request: + payload: {} +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/policies.yaml b/tests/default/ism/policies.yaml index 0a2f75e7f..ae88b823b 100644 --- a/tests/default/ism/policies.yaml +++ b/tests/default/ism/policies.yaml @@ -1,10 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index state management operations. -epilogues: - - path: /_plugins/_ism/policies/rollover - method: DELETE - status: [200, 404] chapters: - synopsis: Create a rollover policy. id: policy @@ -48,16 +44,27 @@ chapters: output: primary_term: payload._primary_term seq_no: payload._seq_no + - synopsis: Check whether a policy exists. + path: /_plugins/_ism/policies/{policy_id} + method: HEAD + parameters: + policy_id: rollover + response: + status: 200 - synopsis: Get a policy. path: /_plugins/_ism/policies/{policy_id} method: GET parameters: - policy_id: rollover - - synopsis: Update a policy. - path: /_plugins/_ism/policies/{policy_id} + policy_id: rollover + - synopsis: Get all policies. + path: /_plugins/_ism/policies + method: GET + - synopsis: Update a policy specified in the query. + id: updated_policy + path: /_plugins/_ism/policies method: PUT parameters: - policy_id: rollover + policyID: rollover if_primary_term: ${policy.primary_term} if_seq_no: ${policy.seq_no} request: @@ -72,6 +79,28 @@ chapters: min_doc_count: 3 response: status: 200 + output: + primary_term: payload._primary_term + seq_no: payload._seq_no + - synopsis: Update a policy specified in the path. + path: /_plugins/_ism/policies/{policy_id} + method: PUT + parameters: + policy_id: rollover + if_primary_term: ${updated_policy.primary_term} + if_seq_no: ${updated_policy.seq_no} + request: + payload: + policy: + description: Ingesting rollover. + default_state: ingest + states: + - name: ingest + actions: + - rollover: + min_doc_count: 3 + response: + status: 200 - synopsis: Delete a policy. path: /_plugins/_ism/policies/{policy_id} method: DELETE @@ -79,3 +108,7 @@ chapters: policy_id: rollover response: status: 200 +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/refresh_search_analyzers.yaml b/tests/default/ism/refresh_search_analyzers.yaml new file mode 100644 index 000000000..532a0f498 --- /dev/null +++ b/tests/default/ism/refresh_search_analyzers.yaml @@ -0,0 +1,43 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Refresh search analyzers. +prologues: + - path: /books + method: PUT + request: + payload: + settings: + analysis: + analyzer: + movie_titles: + tokenizer: whitespace + filter: + - movie_titles + filter: + movie_titles: + type: synonym_graph + synonyms_path: opensearch.yml # synonyms.txt + updateable: true + mappings: + properties: + title: + type: text + analyzer: simple + search_analyzer: movie_titles +chapters: + - synopsis: Refresh search analyzers for an index. + path: /_plugins/_refresh_search_analyzers/{index} + method: POST + parameters: + index: books + response: + status: 200 + payload: + successful_refresh_details: + - index: books + refreshed_analyzers: + - movie_titles +epilogues: + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/remove.yaml b/tests/default/ism/remove.yaml index f688cd519..93f673ee7 100644 --- a/tests/default/ism/remove.yaml +++ b/tests/default/ism/remove.yaml @@ -1,13 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml -description: Test adding a policy to an index. -epilogues: - - path: /_plugins/_ism/policies/rollover - method: DELETE - status: [200, 404] - - path: /books - method: DELETE - status: [200, 404] +description: Test removing a policy from an index. prologues: - path: /books method: PUT @@ -33,7 +26,12 @@ prologues: payload: policy_id: ${policy.id} chapters: - - synopsis: Remove the current policy from an index. + - synopsis: Remove the current policy from an index specified in the query. + path: /_plugins/_ism/remove + method: POST + parameters: + index: books + - synopsis: Remove the current policy from an index specified in the path. path: /_plugins/_ism/remove/{index} method: POST parameters: @@ -43,3 +41,10 @@ chapters: method: POST parameters: index: books +epilogues: + - path: /_plugins/_ism/policies/rollover + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/ism/retry.yaml b/tests/default/ism/retry.yaml new file mode 100644 index 000000000..6d5a3312f --- /dev/null +++ b/tests/default/ism/retry.yaml @@ -0,0 +1,43 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test retrying a failed action for an index. +prologues: + - path: /books + method: PUT +chapters: + - synopsis: Retry a failed policy for an index specified in the query. + path: /_plugins/_ism/retry + method: POST + parameters: + index: books + request: + payload: + state: index + response: + status: 200 + payload: + updated_indices: 0 + failures: true + failed_indices: + - index_name: books + reason: This index is not being managed. + - synopsis: Retry a failed policy for an index specified in the path. + path: /_plugins/_ism/retry/{index} + method: POST + parameters: + index: books + request: + payload: + state: index + response: + status: 200 + payload: + updated_indices: 0 + failures: true + failed_indices: + - index_name: books + reason: This index is not being managed. +epilogues: + - path: /books + method: DELETE + status: [200, 404] From 2487252d82c1007ad0bf2edf434c998374469c57 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Fri, 20 Sep 2024 04:32:54 +1200 Subject: [PATCH 24/73] Fix routing param type and metric path params (#586) --- CHANGELOG.md | 5 +++++ spec/namespaces/_core.yaml | 19 ++++++++++++++----- spec/namespaces/cluster.yaml | 6 +++++- spec/namespaces/indices.yaml | 10 +++++++--- spec/namespaces/knn.yaml | 5 +---- spec/schemas/_common.yaml | 12 +++++------- spec/schemas/cluster.reroute.yaml | 12 ++++++++++++ spec/schemas/indices.shard_stores.yaml | 2 +- spec/schemas/indices.stats.yaml | 21 +++++++++++++++++++++ 9 files changed, 71 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0129e440f..6c7423860 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833)) - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) - Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) +- Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) ### Changed @@ -114,6 +115,10 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Rename `Bytes` component to `StorageType` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Rename `ByteSize` to `StorageSize` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Split `Percentage` into `PercentageNumber`, `PercentageString` ([#573](https://github.com/opensearch-project/opensearch-api-specification/pull/573)) +- Unified all `routing` query params to take a string or array of strings ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) +- Changed `rank_eval`'s `search_type` query param to use the `SearchType` enum rather than a plain string ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) +- Changed `cluster.reroute`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) +- Changed `indices.stats`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) ### Deprecated diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 3c2187e78..0ef3a16a6 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -560,6 +560,7 @@ paths: parameters: - $ref: '#/components/parameters/put_script::path.id' - $ref: '#/components/parameters/put_script::query.cluster_manager_timeout' + - $ref: '#/components/parameters/put_script::query.context' - $ref: '#/components/parameters/put_script::query.master_timeout' - $ref: '#/components/parameters/put_script::query.timeout' requestBody: @@ -577,6 +578,7 @@ paths: parameters: - $ref: '#/components/parameters/put_script::path.id' - $ref: '#/components/parameters/put_script::query.cluster_manager_timeout' + - $ref: '#/components/parameters/put_script::query.context' - $ref: '#/components/parameters/put_script::query.master_timeout' - $ref: '#/components/parameters/put_script::query.timeout' requestBody: @@ -611,6 +613,7 @@ paths: - $ref: '#/components/parameters/put_script::path.context' - $ref: '#/components/parameters/put_script::path.id' - $ref: '#/components/parameters/put_script::query.cluster_manager_timeout' + - $ref: '#/components/parameters/put_script::query.context' - $ref: '#/components/parameters/put_script::query.master_timeout' - $ref: '#/components/parameters/put_script::query.timeout' requestBody: @@ -629,6 +632,7 @@ paths: - $ref: '#/components/parameters/put_script::path.context' - $ref: '#/components/parameters/put_script::path.id' - $ref: '#/components/parameters/put_script::query.cluster_manager_timeout' + - $ref: '#/components/parameters/put_script::query.context' - $ref: '#/components/parameters/put_script::query.master_timeout' - $ref: '#/components/parameters/put_script::query.timeout' requestBody: @@ -3478,10 +3482,7 @@ components: description: Comma-separated list of specific routing values. style: form schema: - type: array - items: - type: string - description: Comma-separated list of specific routing values. + $ref: '../schemas/_common.yaml#/components/schemas/Routing' explode: true delete::path.id: in: path @@ -4863,6 +4864,14 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/Duration' x-version-added: '2.0' + put_script::query.context: + name: context + in: query + description: |- + Context in which the script or search template should run. + To prevent errors, the API immediately compiles the script or template in this context. + schema: + type: string put_script::query.master_timeout: in: query name: master_timeout @@ -4920,7 +4929,7 @@ components: name: search_type description: Search operation type schema: - type: string + $ref: '../schemas/_common.yaml#/components/schemas/SearchType' style: form reindex::query.max_docs: name: max_docs diff --git a/spec/namespaces/cluster.yaml b/spec/namespaces/cluster.yaml index 3157459e0..8820d2eda 100644 --- a/spec/namespaces/cluster.yaml +++ b/spec/namespaces/cluster.yaml @@ -1274,7 +1274,11 @@ components: name: metric description: Limits the information returned to the specified metrics. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Metrics' + oneOf: + - $ref: '../schemas/cluster.reroute.yaml#/components/schemas/Metric' + - type: array + items: + $ref: '../schemas/cluster.reroute.yaml#/components/schemas/Metric' style: form cluster.reroute::query.retry_failed: in: query diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index b3cc12eb7..dcd758619 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -4773,10 +4773,10 @@ components: description: List of shard health statuses used to limit the request. schema: oneOf: - - $ref: '../schemas/indices.shard_stores.yaml#/components/schemas/ShardStoreStatus' + - $ref: '../schemas/indices.shard_stores.yaml#/components/schemas/Status' - type: array items: - $ref: '../schemas/indices.shard_stores.yaml#/components/schemas/ShardStoreStatus' + $ref: '../schemas/indices.shard_stores.yaml#/components/schemas/Status' style: form indices.shrink::path.index: in: path @@ -5035,7 +5035,11 @@ components: description: Limit the information returned the specific metrics. required: true schema: - $ref: '../schemas/_common.yaml#/components/schemas/Metrics' + oneOf: + - $ref: '../schemas/indices.stats.yaml#/components/schemas/Metric' + - type: array + items: + $ref: '../schemas/indices.stats.yaml#/components/schemas/Metric' style: simple indices.stats::query.completion_fields: in: query diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index d960f34a5..49352df55 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -500,10 +500,7 @@ components: description: Comma-separated list of specific routing values. style: form schema: - type: array - items: - type: string - description: Comma-separated list of specific routing values. + $ref: '../schemas/_common.yaml#/components/schemas/Routing' explode: true knn.search_models::query.scroll: name: scroll diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 140600787..08f8d68e4 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -356,7 +356,11 @@ components: ScrollId: type: string Routing: - type: string + oneOf: + - type: string + - type: array + items: + type: string DateTime: description: |- A date and time, either as a string whose format can depend on the context (defaulting to ISO 8601), or a @@ -919,12 +923,6 @@ components: - urgent DataStreamName: type: string - Metrics: - oneOf: - - type: string - - type: array - items: - type: string CompletionStats: type: object properties: diff --git a/spec/schemas/cluster.reroute.yaml b/spec/schemas/cluster.reroute.yaml index d6a6c9d0e..2e569fae8 100644 --- a/spec/schemas/cluster.reroute.yaml +++ b/spec/schemas/cluster.reroute.yaml @@ -82,6 +82,18 @@ components: - index - node - shard + Metric: + type: string + enum: + - _all + - blocks + - cluster_manager_node + - master_node + - metadata + - nodes + - routing_nodes + - routing_table + - version RerouteExplanation: type: object properties: diff --git a/spec/schemas/indices.shard_stores.yaml b/spec/schemas/indices.shard_stores.yaml index a6a1a6f2e..8e4268a58 100644 --- a/spec/schemas/indices.shard_stores.yaml +++ b/spec/schemas/indices.shard_stores.yaml @@ -6,7 +6,7 @@ info: paths: {} components: schemas: - ShardStoreStatus: + Status: type: string enum: - all diff --git a/spec/schemas/indices.stats.yaml b/spec/schemas/indices.stats.yaml index b2391cddb..cb13ae0b3 100644 --- a/spec/schemas/indices.stats.yaml +++ b/spec/schemas/indices.stats.yaml @@ -109,6 +109,27 @@ components: $ref: '#/components/schemas/ShardRetentionLeases' shard_path: $ref: '#/components/schemas/ShardPath' + Metric: + type: string + enum: + - _all + - completion + - docs + - fielddata + - flush + - get + - indexing + - merge + - query_cache + - recovery + - refresh + - request_cache + - search + - segments + - store + - suggest + - translog + - warmer ShardCommit: type: object properties: From c6c0c417b19dd9b63c4e37379b7e94d412be7ea8 Mon Sep 17 00:00:00 2001 From: Junwei Dai <59641585+junweid62@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:00:58 -0700 Subject: [PATCH 25/73] Added `_ml/agents` and `ml/connectors` specs (#585) * Add Ml api Spec , /connector/_register and /connector/{connector_id} with test Signed-off-by: Junwei Dai * Add Ml api Spec , /agents/_register and /agents/{agent_id} with test Signed-off-by: Junwei Dai * fix: agent_id error Signed-off-by: Junwei Dai * Update CHANGELOG.md Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> * Readded empty line Signed-off-by: Junwei Dai --------- Signed-off-by: Junwei Dai Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> Co-authored-by: Junwei Dai Co-authored-by: Daniel (dB.) Doubrovkine --- CHANGELOG.md | 1 + spec/namespaces/ml.yaml | 94 +++++++++++++++++++- spec/schemas/ml._common.yaml | 128 ++++++++++++++++++++++++++++ tests/plugins/ml/ml/agents.yaml | 50 +++++++++++ tests/plugins/ml/ml/connectors.yaml | 50 +++++++++++ 5 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 tests/plugins/ml/ml/agents.yaml create mode 100644 tests/plugins/ml/ml/connectors.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7423860..8b89539a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833)) - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) - Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) +- Added `/_plugins/_ml/agents/_register`, `/_plugins/_ml/connectors/_create`, `DELETE /_plugins/_ml/agents/{agent_id}`, `DELETE /_plugins/_ml/connectors/{connector_id}` ([#228](https://github.com/opensearch-project/opensearch-api-specification/issues/228)) - Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) ### Changed diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 17ee03088..700fc334f 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -102,6 +102,46 @@ paths: responses: '200': $ref: '#/components/responses/ml.search_models@200' + /_plugins/_ml/connectors/_create: + post: + operationId: ml.create_connector.0 + x-operation-group: ml.create_connector + description: Creates a standalone connector. + requestBody: + $ref: '#/components/requestBodies/ml.create_connector' + responses: + '200': + $ref: '#/components/responses/ml.create_connector@200' + /_plugins/_ml/connectors/{connector_id}: + delete: + operationId: ml.delete_connector.0 + x-operation-group: ml.delete_connector + description: Deletes a standalone connector. + parameters: + - $ref: '#/components/parameters/ml.delete_connector::path.connector_id' + responses: + '200': + $ref: '#/components/responses/ml.delete_connector@200' + /_plugins/_ml/agents/_register: + post: + operationId: ml.register_agents.0 + x-operation-group: ml.register_agents + description: Register an agent. + requestBody: + $ref: '#/components/requestBodies/ml.register_agents' + responses: + '200': + $ref: '#/components/responses/ml.register_agents@200' + /_plugins/_ml/agents/{agent_id}: + delete: + operationId: ml.delete_agent.0 + x-operation-group: ml.delete_agent + description: Delete an agent. + parameters: + - $ref: '#/components/parameters/ml.delete_agent::path.agent_id' + responses: + '200': + $ref: '#/components/responses/ml.delete_agent@200' components: requestBodies: ml.register_model_group: @@ -110,7 +150,7 @@ components: schema: type: object properties: - name: + name: type: string description: The model group name. description: @@ -136,7 +176,7 @@ components: schema: type: object properties: - name: + name: type: string description: The model name. version: @@ -161,6 +201,16 @@ components: application/json: schema: $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsQuery' + ml.create_connector: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/CreateConnectorRequest' + ml.register_agents: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/RegisterAgentsRequest' responses: ml.register_model_group@200: content: @@ -230,6 +280,32 @@ components: application/json: schema: $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsResponse' + ml.create_connector@200: + content: + application/json: + schema: + type: object + properties: + connector_id: + type: string + ml.delete_connector@200: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' + ml.register_agents@200: + content: + application/json: + schema: + type: object + properties: + agent_id: + type: string + ml.delete_agent@200: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' parameters: ml.get_model_group::path.model_group_id: name: model_group_id @@ -267,9 +343,21 @@ components: required: true schema: type: string + ml.delete_connector::path.connector_id: + name: connector_id + in: path + required: true + schema: + type: string + ml.delete_agent::path.agent_id: + name: agent_id + in: path + required: true + schema: + type: string ml.get_task::path.task_id: name: task_id in: path required: true schema: - type: string + type: string \ No newline at end of file diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index e95621511..46cffc9f3 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -156,3 +156,131 @@ components: UndeployModelNodeStats: type: object additionalProperties: true + CreateConnectorRequest: + type: object + properties: + name: + type: string + description: + type: string + version: + type: integer + protocol: + type: string + credential: + $ref: '#/components/schemas/Credential' + parameters: + $ref: '#/components/schemas/Parameters' + client_config: + $ref: '#/components/schemas/ClientConfig' + actions: + type: array + items: + $ref: '#/components/schemas/Action' + required: + - actions + - credential + - description + - name + - parameters + - protocol + - version + Credential: + type: object + properties: + access_key: + type: string + secret_key: + type: string + session_token: + type: string + additionalProperties: true + Parameters: + type: object + additionalProperties: true + ClientConfig: + type: object + properties: + max_connection: + type: integer + connection_timeout: + type: integer + read_timeout: + type: integer + retry_backoff_policy: + type: string + max_retry_times: + type: integer + retry_backoff_millis: + type: integer + retry_timeout_seconds: + type: integer + Action: + type: object + properties: + action_type: + type: string + method: + type: string + headers: + $ref: '#/components/schemas/Headers' + url: + type: string + request_body: + type: string + pre_process_function: + type: string + post_process_function: + type: string + RegisterAgentsRequest: + type: object + properties: + name: + type: string + type: + type: string + description: + type: string + tools: + type: array + items: + $ref: '#/components/schemas/ToolItems' + parameters: + $ref: '#/components/schemas/Parameters' + app_type: + type: string + memory: + $ref: '#/components/schemas/Memory' + llm: + $ref: '#/components/schemas/LLM' + required: + - name + - type + LLM: + type: object + properties: + model_id: + type: string + parameters: + $ref: '#/components/schemas/Parameters' + Memory: + type: object + properties: + type: + type: string + ToolItems: + type: object + properties: + name: + type: string + type: + type: string + parameters: + $ref: '#/components/schemas/Parameters' + additionalProperties: true + Headers: + type: object + properties: + content_type: + type: string + additionalProperties: true \ No newline at end of file diff --git a/tests/plugins/ml/ml/agents.yaml b/tests/plugins/ml/ml/agents.yaml new file mode 100644 index 000000000..be85d3275 --- /dev/null +++ b/tests/plugins/ml/ml/agents.yaml @@ -0,0 +1,50 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the register of Agents. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +epilogues: + - path: /_plugins/_ml/agents/{agent_id} + method: DELETE + status: [200, 404] + parameters: + agent_id: ${register_agent.test_agent_id} +chapters: + - synopsis: Register agent. + warnings: + multiple-paths-detected: false + id: register_agent + path: /_plugins/_ml/agents/_register + method: POST + request: + payload: + name: Test_Agent_For_RAG + type: flow + description: this is a test agent + tools: + - type: VectorDBTool + parameters: + model_id: YOUR_TEXT_EMBEDDING_MODEL_ID + index: my_test_data + embedding_field: embedding + source_field: + - text + input: test_question + - type: MLModelTool + description: A general tool to answer any question + parameters: + model_id: YOUR_LLM_MODEL_ID + response: + status: 200 + output: + test_agent_id: payload.agent_id + - synopsis: Delete agent. + path: /_plugins/_ml/agents/{agent_id} + method: DELETE + parameters: + agent_id: ${register_agent.test_agent_id} + response: + status: 200 \ No newline at end of file diff --git a/tests/plugins/ml/ml/connectors.yaml b/tests/plugins/ml/ml/connectors.yaml new file mode 100644 index 000000000..201019c04 --- /dev/null +++ b/tests/plugins/ml/ml/connectors.yaml @@ -0,0 +1,50 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation of connectors. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +epilogues: + - path: /_plugins/_ml/connectors/{connector_id} + method: DELETE + status: [200, 404] + parameters: + connector_id: ${create_connector.test_connector_id} +chapters: + - synopsis: Create connector. + warnings: + multiple-paths-detected: false + id: create_connector + path: /_plugins/_ml/connectors/_create + method: POST + request: + payload: + name: OpenAI Chat Connector + description: The connector to public OpenAI model service for GPT 3.5 + version: 1 + protocol: http + parameters: + endpoint: api.openai.com + model: gpt-3.5-turbo + credential: + openAI_key: test_api_key + actions: + - action_type: predict + method: POST + url: 'https://${parameters.endpoint}/v1/chat/completions' + headers: + Authorization: 'Bearer ${credential.openAI_key}' + request_body: '{ "model": "${parameters.model}", "messages": ${parameters.messages} }' + response: + status: 200 + output: + test_connector_id: payload.connector_id + - synopsis: Delete connector. + path: /_plugins/_ml/connectors/{connector_id} + method: DELETE + parameters: + connector_id: ${create_connector.test_connector_id} + response: + status: 200 \ No newline at end of file From 4f8be9eb9a62ea86a3a864f732c6fc126e4554a6 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 25 Sep 2024 00:21:48 +1200 Subject: [PATCH 26/73] Fix CleanupRepositoryResults properties to be longs (#587) * Fix CleanupRepositoryResults properties to be longs Signed-off-by: Thomas Farr * changelog Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/schemas/snapshot.cleanup_repository.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b89539a7..e76107eac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Changed `rank_eval`'s `search_type` query param to use the `SearchType` enum rather than a plain string ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) - Changed `cluster.reroute`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) - Changed `indices.stats`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) +- Changed `CleanupRepositoryResults`' properties to be `int64`s ([#587](https://github.com/opensearch-project/opensearch-api-specification/pull/587)) ### Deprecated diff --git a/spec/schemas/snapshot.cleanup_repository.yaml b/spec/schemas/snapshot.cleanup_repository.yaml index 0682a8839..de41cadb0 100644 --- a/spec/schemas/snapshot.cleanup_repository.yaml +++ b/spec/schemas/snapshot.cleanup_repository.yaml @@ -11,10 +11,12 @@ components: properties: deleted_blobs: description: Number of binary large objects (blobs) removed during cleanup. - type: number + type: integer + format: int64 deleted_bytes: description: Number of bytes freed by cleanup operations. - type: number + type: integer + format: int64 required: - deleted_blobs - deleted_bytes From 7d0566403af3fa7e1898e2eee34ee83ec5cd0866 Mon Sep 17 00:00:00 2001 From: Ryan Bogan Date: Tue, 24 Sep 2024 12:44:09 -0700 Subject: [PATCH 27/73] Update API specifications for k-NN 2.17 changes (#588) * Update API specifications for k-NN 2.17 changes Signed-off-by: Ryan Bogan * Add changelog entry Signed-off-by: Ryan Bogan * Refactor k-NN tests, change naming to match theme, and update changelog message Signed-off-by: Ryan Bogan * Add spaceType parameter to train model API Signed-off-by: Ryan Bogan * Update sha references and remove version checks for asynchronous search Signed-off-by: Ryan Bogan * Add version checks back for asynchronous search Signed-off-by: Ryan Bogan * Update changelog style for entry Signed-off-by: Ryan Bogan --------- Signed-off-by: Ryan Bogan --- .github/workflows/test-spec.yml | 5 +- CHANGELOG.md | 1 + spec/namespaces/knn.yaml | 18 ++++- spec/schemas/_common.mapping.yaml | 8 ++ spec/schemas/_common.yaml | 10 +++ tests/default/_core/search/knn/on_disk.yaml | 71 ++++++++++++++++++ .../search/{knn.yaml => knn/search.yaml} | 2 +- tests/default/knn/train_model.yaml | 73 +++++++++++++++++++ 8 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 tests/default/_core/search/knn/on_disk.yaml rename tests/default/_core/search/{knn.yaml => knn/search.yaml} (98%) create mode 100644 tests/default/knn/train_model.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a202207de..8f15414e3 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,11 +37,12 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.17.0 + - version: 2.18.0 hub: opensearchstaging - ref: '@sha256:1273489ebbedcb470ea13563dae4c6dc6b2ed431e87e686ed931ae0733034b25' + ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:06af2ba4037f8423dc1a4ed3cd29108a1912774e7c659e73f0fac09e1bb2b63d' + ref: '@sha256:cf07c0ffa7d9e8a3e7fdb58041caae3bb81f1123260431b99d0ebf4a52c3d9a3' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index e76107eac..9f7833649 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -165,6 +165,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) +- Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) ### Security diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index 49352df55..a33bd02df 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -268,11 +268,16 @@ components: format: int32 description: type: string + mode: + type: string + compression_level: + type: string method: type: string + spaceType: + type: string required: - dimension - - method - training_field - training_index required: true @@ -281,7 +286,16 @@ components: knn.get_model@200: {} knn.search_models@200: {} knn.stats@200: {} - knn.train_model@200: {} + knn.train_model@200: + content: + application/json: + schema: + type: object + properties: + model_id: + type: string + required: + - model_id knn.warmup@200: {} parameters: knn.delete_model::path.model_id: diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index a26c15e61..2c26487fb 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -1141,6 +1141,14 @@ components: properties: dimension: type: number + space_type: + type: string + data_type: + type: string + mode: + type: string + compression_level: + type: string method: $ref: '#/components/schemas/KnnVectorMethod' required: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 08f8d68e4..889c8b014 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -660,6 +660,16 @@ components: boost: description: Boost value to apply to kNN scores type: number + method_parameters: + type: object + x-version-added: '2.16' + additionalProperties: + type: number + rescore: + type: object + x-version-added: '2.17' + additionalProperties: + type: number required: - vector QueryVector: diff --git a/tests/default/_core/search/knn/on_disk.yaml b/tests/default/_core/search/knn/on_disk.yaml new file mode 100644 index 000000000..ac42d42f4 --- /dev/null +++ b/tests/default/_core/search/knn/on_disk.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with knn query. +version: '>= 2.17' + +prologues: + - method: PUT + path: /movies + request: + payload: + settings: + index: + knn: true + mappings: + properties: + recommendation_vector: + type: knn_vector + dimension: 8 + space_type: l2 + data_type: float + mode: on_disk + compression_level: 16x + status: [200] + - method: POST + path: /_bulk + request: + content_type: application/x-ndjson + payload: + - {index: {_index: movies, _id: '1'}} + - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} + - {index: {_index: movies, _id: '2'}} + - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 12.9} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 1.2} + - {index: {_index: movies, _id: '5'}} + - {recommendation_vector: [5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5], duration: 3.7} + - {index: {_index: movies, _id: '6'}} + - {recommendation_vector: [6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5], duration: 10.3} + - {index: {_index: movies, _id: '7'}} + - {recommendation_vector: [7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5], duration: 5.5} + - {index: {_index: movies, _id: '8'}} + - {recommendation_vector: [8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5], duration: 4.4} + - {index: {_index: movies, _id: '9'}} + - {recommendation_vector: [9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5], duration: 8.9} + status: [200] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Test k-NN disk-based search. + method: POST + path: /{index}/_search + parameters: + index: movies + request: + payload: + query: + knn: + recommendation_vector: + vector: [1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5] + k: 5 + method_parameters: + ef_search: 512 + rescore: + oversample_factor: 10 + response: + status: 200 \ No newline at end of file diff --git a/tests/default/_core/search/knn.yaml b/tests/default/_core/search/knn/search.yaml similarity index 98% rename from tests/default/_core/search/knn.yaml rename to tests/default/_core/search/knn/search.yaml index 6fcfb12e6..ae4daedeb 100644 --- a/tests/default/_core/search/knn.yaml +++ b/tests/default/_core/search/knn/search.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with knn query. version: '>= 1.2' diff --git a/tests/default/knn/train_model.yaml b/tests/default/knn/train_model.yaml new file mode 100644 index 000000000..b45ac5c6f --- /dev/null +++ b/tests/default/knn/train_model.yaml @@ -0,0 +1,73 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test training k-NN model with disk-based parameters. +version: '>= 2.17' + +prologues: + - method: PUT + path: /movies + request: + payload: + settings: + index: + knn: true + mappings: + properties: + recommendation_vector: + type: knn_vector + dimension: 8 + status: [200] + - method: POST + path: /_bulk + request: + content_type: application/x-ndjson + payload: + - {index: {_index: movies, _id: '1'}} + - {recommendation_vector: [1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5, 1.5], duration: 12.2} + - {index: {_index: movies, _id: '2'}} + - {recommendation_vector: [2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5, 2.5], duration: 7.1} + - {index: {_index: movies, _id: '3'}} + - {recommendation_vector: [3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5, 3.5], duration: 12.9} + - {index: {_index: movies, _id: '4'}} + - {recommendation_vector: [4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5, 4.5], duration: 1.2} + - {index: {_index: movies, _id: '5'}} + - {recommendation_vector: [5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5, 5.5], duration: 3.7} + - {index: {_index: movies, _id: '6'}} + - {recommendation_vector: [6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5, 6.5], duration: 10.3} + - {index: {_index: movies, _id: '7'}} + - {recommendation_vector: [7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5, 7.5], duration: 5.5} + - {index: {_index: movies, _id: '8'}} + - {recommendation_vector: [8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5, 8.5], duration: 4.4} + - {index: {_index: movies, _id: '9'}} + - {recommendation_vector: [9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5], duration: 8.9} + status: [200] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + - path: /_plugins/_knn/models/{model_id} + parameters: + model_id: ${train_model.test_model_id} + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Test training a model with disk-based parameters. + id: train_model + method: POST + path: /_plugins/_knn/models/_train + request: + payload: + training_index: movies + training_field: recommendation_vector + dimension: 8 + max_training_vector_count: 1200 + search_size: 100 + description: Test model + mode: on_disk + compression_level: 32x + spaceType: l2 + response: + status: 200 + output: + test_model_id: payload.model_id \ No newline at end of file From a92c4923142bb3b53e41a10a374ef46d8d0efdba Mon Sep 17 00:00:00 2001 From: kkewwei Date: Mon, 30 Sep 2024 00:37:40 +0800 Subject: [PATCH 28/73] Added aggregation tests for avg, max, min, range, terms, sum, nested and histogram. (#576) * Add aggregation tests for nested Signed-off-by: kkewwei Signed-off-by: kewei.11 * Added aggregation tests for avg, max, min, range, terms, sum and histogram. Signed-off-by: dblock Signed-off-by: kkewwei --------- Signed-off-by: kkewwei Signed-off-by: kewei.11 Signed-off-by: dblock Co-authored-by: dblock --- CHANGELOG.md | 1 + spec/schemas/_common.aggregations.yaml | 5 +- .../_core/search/aggregations/avg.yaml | 38 ++++++++++++ .../_core/search/aggregations/historgram.yaml | 48 +++++++++++++++ .../_core/search/aggregations/max.yaml | 38 ++++++++++++ .../_core/search/aggregations/min.yaml | 38 ++++++++++++ .../_core/search/aggregations/nested.yaml | 59 +++++++++++++++++++ .../_core/search/aggregations/range.yaml | 46 +++++++++++++++ .../_core/search/aggregations/sum.yaml | 38 ++++++++++++ .../_core/search/aggregations/terms.yaml | 54 +++++++++++++++++ 10 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 tests/default/_core/search/aggregations/avg.yaml create mode 100644 tests/default/_core/search/aggregations/historgram.yaml create mode 100644 tests/default/_core/search/aggregations/max.yaml create mode 100644 tests/default/_core/search/aggregations/min.yaml create mode 100644 tests/default/_core/search/aggregations/nested.yaml create mode 100644 tests/default/_core/search/aggregations/range.yaml create mode 100644 tests/default/_core/search/aggregations/sum.yaml create mode 100644 tests/default/_core/search/aggregations/terms.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f7833649..1081c7af2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) - Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) +- Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) ### Security diff --git a/spec/schemas/_common.aggregations.yaml b/spec/schemas/_common.aggregations.yaml index 3313b096b..be7ef28a4 100644 --- a/spec/schemas/_common.aggregations.yaml +++ b/spec/schemas/_common.aggregations.yaml @@ -7,7 +7,7 @@ paths: {} components: schemas: Aggregate: - oneOf: + anyOf: - $ref: '#/components/schemas/CardinalityAggregate' - $ref: '#/components/schemas/HdrPercentilesAggregate' - $ref: '#/components/schemas/HdrPercentileRanksAggregate' @@ -786,6 +786,9 @@ components: allOf: - $ref: '#/components/schemas/SingleBucketAggregateBase' - type: object + properties: + doc_count: + type: number ReverseNestedAggregate: allOf: - $ref: '#/components/schemas/SingleBucketAggregateBase' diff --git a/tests/default/_core/search/aggregations/avg.yaml b/tests/default/_core/search/aggregations/avg.yaml new file mode 100644 index 000000000..52daa7ecb --- /dev/null +++ b/tests/default/_core/search/aggregations/avg.yaml @@ -0,0 +1,38 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test avg aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, duration: 126} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, duration: 100} +chapters: + - synopsis: Aggregate with avg. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + duration_avg: + avg: + field: duration + response: + status: 200 + payload: + aggregations: + duration_avg: + value: 113 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/historgram.yaml b/tests/default/_core/search/aggregations/historgram.yaml new file mode 100644 index 000000000..03adc94f4 --- /dev/null +++ b/tests/default/_core/search/aggregations/historgram.yaml @@ -0,0 +1,48 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test histogram aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Aggregate with histogram. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + movies: + histogram: + field: year + interval: 100 + format: '##0.0' + response: + status: 200 + payload: + aggregations: + movies: + buckets: + - key_as_string: '1900.0' + key: 1900 + doc_count: 2 + - key_as_string: '2000.0' + key: 2000 + doc_count: 1 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/max.yaml b/tests/default/_core/search/aggregations/max.yaml new file mode 100644 index 000000000..4d05066b2 --- /dev/null +++ b/tests/default/_core/search/aggregations/max.yaml @@ -0,0 +1,38 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test max aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, duration: 126} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, duration: 100} +chapters: + - synopsis: Aggregate with max. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + duration_max: + max: + field: duration + response: + status: 200 + payload: + aggregations: + duration_max: + value: 126 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/min.yaml b/tests/default/_core/search/aggregations/min.yaml new file mode 100644 index 000000000..0d0f6197c --- /dev/null +++ b/tests/default/_core/search/aggregations/min.yaml @@ -0,0 +1,38 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test min aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, duration: 126} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, duration: 100} +chapters: + - synopsis: Aggregate with min. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + duration_min: + min: + field: duration + response: + status: 200 + payload: + aggregations: + duration_min: + value: 100 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/nested.yaml b/tests/default/_core/search/aggregations/nested.yaml new file mode 100644 index 000000000..4b519dcc5 --- /dev/null +++ b/tests/default/_core/search/aggregations/nested.yaml @@ -0,0 +1,59 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test aggregation. +prologues: + - path: books + method: PUT + request: + payload: + mappings: + properties: + details: + type: nested + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {index: {_index: books, _id: '0'}} + - {details: {title: title1}} + - {index: {_index: books, _id: '1'}} + - {details: {title: title2}} +chapters: + - synopsis: Aggregation with nested. + path: /{index}/_search + parameters: + index: books + method: POST + request: + payload: + size: 0 + aggregations: + details_agg: + nested: + path: details + response: + status: 200 + payload: + timed_out: false + _shards: + total: 1 + successful: 1 + skipped: 0 + failed: 0 + hits: + total: + value: 2 + relation: eq + max_score: null + hits: [] + aggregations: + details_agg: + doc_count: 2 + +epilogues: + - path: /books + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/range.yaml b/tests/default/_core/search/aggregations/range.yaml new file mode 100644 index 000000000..0c34de8db --- /dev/null +++ b/tests/default/_core/search/aggregations/range.yaml @@ -0,0 +1,46 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test range aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Aggregate with range. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + 20th_century: + range: + field: year + ranges: + - from: 1900 + to: 2000 + response: + status: 200 + payload: + aggregations: + 20th_century: + buckets: + - from: 1900 + to: 2000 + doc_count: 2 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/sum.yaml b/tests/default/_core/search/aggregations/sum.yaml new file mode 100644 index 000000000..252785ca3 --- /dev/null +++ b/tests/default/_core/search/aggregations/sum.yaml @@ -0,0 +1,38 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test sum aggregation. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, duration: 126} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, duration: 100} +chapters: + - synopsis: Aggregate with sum. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + duration_sum: + sum: + field: duration + response: + status: 200 + payload: + aggregations: + duration_sum: + value: 226 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] diff --git a/tests/default/_core/search/aggregations/terms.yaml b/tests/default/_core/search/aggregations/terms.yaml new file mode 100644 index 000000000..19c4a9606 --- /dev/null +++ b/tests/default/_core/search/aggregations/terms.yaml @@ -0,0 +1,54 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test terms aggregation. +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + director: + type: text + fielddata: true + fields: + raw: + type: keyword + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball} + - {create: {_index: movies}} + - {director: Bennett Miller, title: The Cruise} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive} +chapters: + - synopsis: Aggregate with terms. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + size: 0 + aggregations: + directors: + terms: + field: director.raw + response: + status: 200 + payload: + aggregations: + directors: + buckets: + - key: Bennett Miller + doc_count: 2 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] From ee984193d905ee1913515bcb0ed04dbcd058b552 Mon Sep 17 00:00:00 2001 From: Niyazbek Torekeldi <78027392+Tokesh@users.noreply.github.com> Date: Mon, 30 Sep 2024 17:24:08 +0500 Subject: [PATCH 29/73] adding async search tests for OS 3.0 version (#590) * fixing version in stats api specs Signed-off-by: Tokesh * deleting versions constraints in async search tests and adding latest tag to ci for 3.0 Signed-off-by: Tokesh --------- Signed-off-by: Tokesh --- .github/workflows/test-spec.yml | 2 +- tests/default/asynchronous_search/search.yaml | 5 +---- tests/default/asynchronous_search/stats.yaml | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 8f15414e3..a460dc0c1 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -42,7 +42,7 @@ jobs: ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:cf07c0ffa7d9e8a3e7fdb58041caae3bb81f1123260431b99d0ebf4a52c3d9a3' + ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/tests/default/asynchronous_search/search.yaml b/tests/default/asynchronous_search/search.yaml index 23e22d57c..4397fe525 100644 --- a/tests/default/asynchronous_search/search.yaml +++ b/tests/default/asynchronous_search/search.yaml @@ -27,7 +27,6 @@ chapters: synopsis: Asynchronous Search. path: /_plugins/_asynchronous_search method: POST - version: < 3.0 parameters: index: books keep_on_completion: true @@ -44,7 +43,6 @@ chapters: - synopsis: Get partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: GET - version: < 3.0 parameters: id: ${async_search.id} response: @@ -52,8 +50,7 @@ chapters: - synopsis: Delete partial response from asynchronous search. path: /_plugins/_asynchronous_search/{id} method: DELETE - version: < 3.0 parameters: id: ${async_search.id} response: - status: 200 + status: 200 \ No newline at end of file diff --git a/tests/default/asynchronous_search/stats.yaml b/tests/default/asynchronous_search/stats.yaml index 7854b156e..b5f113006 100644 --- a/tests/default/asynchronous_search/stats.yaml +++ b/tests/default/asynchronous_search/stats.yaml @@ -6,6 +6,5 @@ chapters: - synopsis: Get stats. path: /_plugins/_asynchronous_search/stats method: GET - version: < 3.0 response: status: 200 From ee2d99ca9b40f862fdca35390eda0001c30c1c53 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 30 Sep 2024 11:47:10 -0400 Subject: [PATCH 30/73] Document new experimental ingestion streaming APIs (#584) * Document new experimental ingestion streaming APIs Signed-off-by: Andriy Redko * Address review comments Signed-off-by: Andriy Redko * Address review comments and add tests Signed-off-by: Andriy Redko * Fix version constraints for tests Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko * Add transport-reactor-netty4 plugin to 2.17.0+ test containers Signed-off-by: Andriy Redko * Enable streaming tests for 2.17.0 only Signed-off-by: Andriy Redko * Exclude Dockefile from the list of scenario files Signed-off-by: Andriy Redko * Move tests from tests/default to tests/plugins for streaming Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko --- .github/workflows/test-spec.yml | 1 + .github/workflows/test-tools-integ.yml | 2 +- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 244 ++++++++++++++++++ spec/schemas/_common.yaml | 3 + tests/plugins/streaming/Dockerfile | 6 + .../plugins/streaming/_core/bulk/stream.yaml | 54 ++++ tests/plugins/streaming/docker-compose.yml | 18 ++ .../streaming/indices/bulk/stream.yaml | 60 +++++ tools/src/tester/TestRunner.ts | 2 +- .../tester/fixtures/evals/passed/passed.yaml | 2 +- .../tester/fixtures/evals/skipped/semver.yaml | 2 +- tools/tests/tester/helpers.ts | 2 +- 13 files changed, 392 insertions(+), 5 deletions(-) create mode 100644 tests/plugins/streaming/Dockerfile create mode 100644 tests/plugins/streaming/_core/bulk/stream.yaml create mode 100644 tests/plugins/streaming/docker-compose.yml create mode 100644 tests/plugins/streaming/indices/bulk/stream.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a460dc0c1..06a098265 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,6 +37,7 @@ jobs: - version: 2.16.0 tests: snapshot - version: 2.17.0 + tests: plugins/streaming - version: 2.18.0 hub: opensearchstaging ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index 5bb9ac5d8..d4384f4af 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -24,7 +24,7 @@ jobs: test: runs-on: ubuntu-latest env: - OPENSEARCH_VERSION: 2.16.0 + OPENSEARCH_VERSION: 2.17.0 OPENSEARCH_PASSWORD: myStrongPassword123! OPENSEARCH_URL: https://localhost:9200 steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index 1081c7af2..34f8fa1cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833)) - Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) - Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578)) +- Added `/_bulk/stream` ([#584](https://github.com/opensearch-project/opensearch-api-specification/pull/584)) - Added `/_plugins/_ml/agents/_register`, `/_plugins/_ml/connectors/_create`, `DELETE /_plugins/_ml/agents/{agent_id}`, `DELETE /_plugins/_ml/connectors/{connector_id}` ([#228](https://github.com/opensearch-project/opensearch-api-specification/issues/228)) - Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 0ef3a16a6..a9e7b8b0d 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -76,6 +76,57 @@ paths: responses: '200': $ref: '#/components/responses/bulk@200' + /_bulk/stream: + post: + operationId: bulk_stream.0 + x-operation-group: bulk_stream + x-version-added: '2.17' + description: Allows to perform multiple index/update/delete operations using request response streaming. + externalDocs: + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ + parameters: + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' + requestBody: + $ref: '#/components/requestBodies/bulk_stream' + responses: + '200': + $ref: '#/components/responses/bulk_stream@200' + put: + operationId: bulk_stream.1 + x-operation-group: bulk_stream + x-version-added: '2.17' + description: Allows to perform multiple index/update/delete operations using request response streaming. + externalDocs: + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ + parameters: + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' + requestBody: + $ref: '#/components/requestBodies/bulk_stream' + responses: + '200': + $ref: '#/components/responses/bulk_stream@200' /_count: get: operationId: count.0 @@ -1056,6 +1107,59 @@ paths: responses: '200': $ref: '#/components/responses/bulk@200' + /{index}/_bulk/stream: + post: + operationId: bulk_stream.2 + x-operation-group: bulk_stream + x-version-added: '2.17' + description: Allows to perform multiple index/update/delete operations using request response streaming. + externalDocs: + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ + parameters: + - $ref: '#/components/parameters/bulk_stream::path.index' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' + requestBody: + $ref: '#/components/requestBodies/bulk_stream' + responses: + '200': + $ref: '#/components/responses/bulk_stream@200' + put: + operationId: bulk_stream.3 + x-operation-group: bulk_stream + x-version-added: '2.17' + description: Allows to perform multiple index/update/delete operations using request response streaming. + externalDocs: + url: https://opensearch.org/docs/latest/api-reference/document-apis/bulk-streaming/ + parameters: + - $ref: '#/components/parameters/bulk_stream::path.index' + - $ref: '#/components/parameters/bulk_stream::query._source' + - $ref: '#/components/parameters/bulk_stream::query._source_excludes' + - $ref: '#/components/parameters/bulk_stream::query._source_includes' + - $ref: '#/components/parameters/bulk_stream::query.batch_interval' + - $ref: '#/components/parameters/bulk_stream::query.batch_size' + - $ref: '#/components/parameters/bulk_stream::query.pipeline' + - $ref: '#/components/parameters/bulk_stream::query.refresh' + - $ref: '#/components/parameters/bulk_stream::query.require_alias' + - $ref: '#/components/parameters/bulk_stream::query.routing' + - $ref: '#/components/parameters/bulk_stream::query.timeout' + - $ref: '#/components/parameters/bulk_stream::query.type' + - $ref: '#/components/parameters/bulk_stream::query.wait_for_active_shards' + requestBody: + $ref: '#/components/requestBodies/bulk_stream' + responses: + '200': + $ref: '#/components/responses/bulk_stream@200' /{index}/_count: get: operationId: count.2 @@ -2156,6 +2260,18 @@ components: - type: object description: The operation definition and data (action-data pairs), separated by newlines required: true + bulk_stream: + content: + application/x-ndjson: + schema: + type: array + items: + anyOf: + - $ref: '../schemas/_core.bulk.yaml#/components/schemas/OperationContainer' + - $ref: '../schemas/_core.bulk.yaml#/components/schemas/UpdateAction' + - type: object + description: The operation definition and data (action-data pairs), separated by newlines + required: true clear_scroll: content: application/json: @@ -2651,6 +2767,30 @@ components: - errors - items - took + bulk_stream@200: + content: + application/json: + schema: + type: object + properties: + errors: + type: boolean + items: + type: array + items: + type: object + additionalProperties: + $ref: '../schemas/_core.bulk.yaml#/components/schemas/ResponseItem' + minProperties: 1 + maxProperties: 1 + took: + type: number + ingest_took: + type: number + required: + - errors + - items + - took clear_scroll@200: content: application/json: @@ -3221,6 +3361,110 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards' style: form x-default: '1' + bulk_stream::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + bulk_stream::query._source: + in: query + name: _source + description: '`true` or `false` to return the `_source` field or not, or a list of fields to return.' + schema: + $ref: '../schemas/_core.search.yaml#/components/schemas/SourceConfigParam' + style: form + bulk_stream::query._source_excludes: + in: query + name: _source_excludes + description: A comma-separated list of source fields to exclude from the response. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Fields' + style: form + bulk_stream::query._source_includes: + in: query + name: _source_includes + description: A comma-separated list of source fields to include in the response. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Fields' + style: form + bulk_stream::query.pipeline: + in: query + name: pipeline + description: |- + ID of the pipeline to use to preprocess incoming documents. + If the index has a default ingest pipeline specified, then setting the value to `_none` disables the default ingest pipeline for this request. + If a final pipeline is configured it will always run, regardless of the value of this parameter. + schema: + type: string + style: form + bulk_stream::query.refresh: + in: query + name: refresh + description: |- + If `true`, OpenSearch refreshes the affected shards to make this operation visible to search, if `wait_for` then wait for a refresh to make this operation visible to search, if `false` do nothing with refreshes. + Valid values: `true`, `false`, `wait_for`. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Refresh' + style: form + bulk_stream::query.require_alias: + in: query + name: require_alias + description: If `true`, the request's actions must target an index alias. + schema: + type: boolean + default: false + style: form + bulk_stream::query.routing: + in: query + name: routing + description: Custom value used to route operations to a specific shard. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Routing' + style: form + bulk_stream::query.timeout: + in: query + name: timeout + description: 'Period each action waits for the following operations: automatic index creation, dynamic mapping updates, waiting for active shards.' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + style: form + bulk_stream::query.type: + name: type + in: query + description: Default document type for items which don't provide one. + schema: + type: string + description: Default document type for items which don't provide one. + bulk_stream::query.wait_for_active_shards: + in: query + name: wait_for_active_shards + description: |- + The number of shard copies that must be active before proceeding with the operation. + Set to all or any positive integer up to the total number of shards in the index (`number_of_replicas+1`). + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WaitForActiveShards' + style: form + x-default: '1' + bulk_stream::query.batch_size: + in: query + name: batch_size + description: |- + Specifies how many bulk operations should be accumulated into a batch before sending the batch to data nodes. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/BatchSize' + style: form + x-default: 1 + bulk_stream::query.batch_interval: + in: query + name: batch_interval + description: |- + Specifies for how long bulk operations should be accumulated into a batch before sending the batch to data nodes. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + style: form clear_scroll::path.scroll_id: in: path name: scroll_id diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 889c8b014..c3e9468d3 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -2238,3 +2238,6 @@ components: required: - reason - status + BatchSize: + type: integer + format: int64 diff --git a/tests/plugins/streaming/Dockerfile b/tests/plugins/streaming/Dockerfile new file mode 100644 index 000000000..ba94fcedd --- /dev/null +++ b/tests/plugins/streaming/Dockerfile @@ -0,0 +1,6 @@ +ARG OPENSEARCH_DOCKER_HUB_PROJECT +ARG OPENSEARCH_VERSION +ARG OPENSEARCH_DOCKER_REF + +FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF} +RUN ./bin/opensearch-plugin install -b transport-reactor-netty4 diff --git a/tests/plugins/streaming/_core/bulk/stream.yaml b/tests/plugins/streaming/_core/bulk/stream.yaml new file mode 100644 index 000000000..7560ea715 --- /dev/null +++ b/tests/plugins/streaming/_core/bulk/stream.yaml @@ -0,0 +1,54 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test bulk streaming endpoint. +epilogues: + - path: /books,movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index. + version: '>= 2.17' + path: /_bulk/stream + method: POST + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - synopsis: Delete document in an index. + version: '2.17' + path: /_bulk/stream + method: PUT + request: + content_type: application/x-ndjson + payload: + - {delete: {_index: movies, _id: invalid}} + response: + status: 200 + payload: + errors: false + items: + - delete: + _index: movies + _id: invalid + result: not_found + status: 404 + - synopsis: Bulk document CRUD. + version: '>= 2.17' + path: /_bulk/stream + method: POST + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {update: {_index: books, _id: book_1392214}} + - {doc: {pages: 376}} + - {update: {_index: books, _id: book_1392214}} + - {doc: {pages: 376}, _source: true} + - {update: {_index: books, _id: book_1392214}} + - {script: {source: ctx._source.pages = 376;}} + - {update: {_index: books, _id: does_not_exist}} + - {script: {source: 'ctx.op = "none";'}, scripted_upsert: true, upsert: {pages: 375}} + - {delete: {_index: books, _id: book_1392214}} + diff --git a/tests/plugins/streaming/docker-compose.yml b/tests/plugins/streaming/docker-compose.yml new file mode 100644 index 000000000..aaf1d9f5b --- /dev/null +++ b/tests/plugins/streaming/docker-compose.yml @@ -0,0 +1,18 @@ +version: '3' + +services: + opensearch-cluster: + build: + context: . + args: + - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject} + - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF} + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node + - http.type=reactor-netty4-secure diff --git a/tests/plugins/streaming/indices/bulk/stream.yaml b/tests/plugins/streaming/indices/bulk/stream.yaml new file mode 100644 index 000000000..546d77b63 --- /dev/null +++ b/tests/plugins/streaming/indices/bulk/stream.yaml @@ -0,0 +1,60 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test bulk streaming index endpoint. +epilogues: + - path: /books,movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index. + version: '2.17' + path: /{index}/_bulk/stream + method: POST + parameters: + index: movies + request: + content_type: application/x-ndjson + payload: + - {create: {}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - synopsis: Delete document in an index. + version: '>= 2.17' + path: /{index}/_bulk/stream + method: PUT + parameters: + index: movies + request: + content_type: application/x-ndjson + payload: + - {delete: {_id: invalid}} + response: + status: 200 + payload: + errors: false + items: + - delete: + _index: movies + _id: invalid + result: not_found + status: 404 + - synopsis: Bulk index document CRUD. + version: '>= 2.17' + method: POST + path: /{index}/_bulk/stream + parameters: + index: books + request: + content_type: application/x-ndjson + payload: + - {create: {_id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {update: {_id: book_1392214}} + - {doc: {pages: 376}} + - {update: {_id: book_1392214}} + - {doc: {pages: 376}, _source: true} + - {update: {_id: book_1392214}} + - {script: {source: ctx._source.pages = 376;}} + - {update: {_id: does_not_exist}} + - {script: {source: 'ctx.op = "none";'}, scripted_upsert: true, upsert: {pages: 375}} + - {delete: {_id: book_1392214}} + diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index dc880f946..7b36af3b9 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -70,7 +70,7 @@ export default class TestRunner { #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (file.startsWith('.') || file == 'docker-compose.yml') { + if (file.startsWith('.') || file == 'docker-compose.yml' || file == 'Dockerfile') { return [] } else if (fs.statSync(path).isFile()) { const story: Story = read_yaml(path) diff --git a/tools/tests/tester/fixtures/evals/passed/passed.yaml b/tools/tests/tester/fixtures/evals/passed/passed.yaml index da08a29fa..a87c944c9 100644 --- a/tools/tests/tester/fixtures/evals/passed/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed/passed.yaml @@ -202,7 +202,7 @@ chapters: - title: This GET /_cat/health should be skipped (> 2.999.0). overall: result: SKIPPED - message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. + message: Skipped because version 2.17.0 does not satisfy >= 2.999.0. - title: This GET /_cat/health should run (>= 1.3, < 99.0). overall: result: PASSED diff --git a/tools/tests/tester/fixtures/evals/skipped/semver.yaml b/tools/tests/tester/fixtures/evals/skipped/semver.yaml index 30709742d..799717afe 100644 --- a/tools/tests/tester/fixtures/evals/skipped/semver.yaml +++ b/tools/tests/tester/fixtures/evals/skipped/semver.yaml @@ -3,5 +3,5 @@ full_path: tools/tests/tester/fixtures/stories/skipped/semver.yaml result: SKIPPED description: This story should be skipped because of version. -message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. +message: Skipped because version 2.17.0 does not satisfy >= 2.999.0. diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 1a38c4c6b..354fcabe4 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -142,5 +142,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.16.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) + }, process.env.OPENSEARCH_VERSION ?? '2.17.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } From 39efae236e80f7b286dee0fe89a7362f450e9a5a Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 30 Sep 2024 11:48:32 -0400 Subject: [PATCH 31/73] Fix: link checker. (#591) --- .github/workflows/check-links.yml | 4 ++-- CHANGELOG.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index d81ca6471..659ec8eb0 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -1,6 +1,6 @@ name: Check Links -on: [pull_request,push] +on: [pull_request, push] jobs: check: @@ -13,4 +13,4 @@ jobs: uses: lycheeverse/lychee-action@v1 with: fail: true - args: --base . --verbose --no-progress './**/*.yaml' './**/*.yml' './**/*.md' './**/*.json' './**/*.ts' --exclude-path ./package-lock.json + args: --base . --verbose --no-progress './**/*.yaml' './**/*.yml' './**/*.md' './**/*.json' './**/*.ts' --exclude-path ./package-lock.json --accept 200,429 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34f8fa1cd..79f9c5354 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -171,4 +171,4 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security -[Unreleased]: https://github.com/opensearch-project/opensearch-api-specification/commits/main/ +[Unreleased]: https://github.com/opensearch-project/opensearch-api-specification/commits/main From 9771b035c205b74eea28a138855bc721e09047c8 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 30 Sep 2024 18:39:18 -0400 Subject: [PATCH 32/73] Added persian_stem. (#592) Signed-off-by: dblock --- .cspell | 7 ++- CHANGELOG.md | 1 + spec/schemas/_common.analysis.yaml | 12 ++++ tests/default/_core/analyze.yaml | 60 ------------------- .../_core/analyze/filter/asciifolding.yaml | 23 +++++++ .../_core/analyze/filter/lowercase.yaml | 24 ++++++++ .../_core/analyze/filter/persian_stem.yaml | 23 +++++++ .../_core/analyze/filter/porterstem.yaml | 23 +++++++ tests/default/_core/analyze/filter/stop.yaml | 26 ++++++++ .../_core/analyze/filter/uppercase.yaml | 23 +++++++ 10 files changed, 160 insertions(+), 62 deletions(-) create mode 100644 tests/default/_core/analyze/filter/asciifolding.yaml create mode 100644 tests/default/_core/analyze/filter/lowercase.yaml create mode 100644 tests/default/_core/analyze/filter/persian_stem.yaml create mode 100644 tests/default/_core/analyze/filter/porterstem.yaml create mode 100644 tests/default/_core/analyze/filter/stop.yaml create mode 100644 tests/default/_core/analyze/filter/uppercase.yaml diff --git a/.cspell b/.cspell index df8953cd6..1576a9ec4 100644 --- a/.cspell +++ b/.cspell @@ -31,6 +31,7 @@ datarows decompounder Decompounder dedup +deprovision determinized distilbert DNFOF @@ -75,6 +76,7 @@ kstem kuromoji Kuromoji languageset +Léon localstats Lovins lucene @@ -137,6 +139,7 @@ Reindex relo reloadcerts remotestore +reprovision rerank Rerank Reranker @@ -192,5 +195,5 @@ vectory whoamiprotected wordnet Yrtsd -reprovision -deprovision \ No newline at end of file +جامد +جامدات \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 79f9c5354..0fb4ed38f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,6 +98,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_bulk/stream` ([#584](https://github.com/opensearch-project/opensearch-api-specification/pull/584)) - Added `/_plugins/_ml/agents/_register`, `/_plugins/_ml/connectors/_create`, `DELETE /_plugins/_ml/agents/{agent_id}`, `DELETE /_plugins/_ml/connectors/{connector_id}` ([#228](https://github.com/opensearch-project/opensearch-api-specification/issues/228)) - Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) +- Added `persian_stem` filter ([#592](https://github.com/opensearch-project/opensearch-api-specification/pull/592)) ### Changed diff --git a/spec/schemas/_common.analysis.yaml b/spec/schemas/_common.analysis.yaml index e917bf0d0..f91d6b4b1 100644 --- a/spec/schemas/_common.analysis.yaml +++ b/spec/schemas/_common.analysis.yaml @@ -474,6 +474,7 @@ components: - $ref: '#/components/schemas/NoriPartOfSpeechTokenFilter' - $ref: '#/components/schemas/PatternCaptureTokenFilter' - $ref: '#/components/schemas/PatternReplaceTokenFilter' + - $ref: '#/components/schemas/PersianStemTokenFilter' - $ref: '#/components/schemas/PorterStemTokenFilter' - $ref: '#/components/schemas/PredicateTokenFilter' - $ref: '#/components/schemas/RemoveDuplicatesTokenFilter' @@ -894,6 +895,17 @@ components: required: - pattern - type + PersianStemTokenFilter: + allOf: + - $ref: '#/components/schemas/TokenFilterBase' + - type: object + properties: + type: + type: string + enum: + - persian_stem + required: + - type PorterStemTokenFilter: allOf: - $ref: '#/components/schemas/TokenFilterBase' diff --git a/tests/default/_core/analyze.yaml b/tests/default/_core/analyze.yaml index 659ec9a96..a5969d930 100644 --- a/tests/default/_core/analyze.yaml +++ b/tests/default/_core/analyze.yaml @@ -30,63 +30,3 @@ chapters: - Moneyball, directed by Bennett Miller response: status: 200 - - synopsis: Apply a filter. - path: /_analyze - method: GET - request: - payload: - tokenizer: keyword - filter: - - uppercase - text: Moneyball - response: - status: 200 - payload: - tokens: - - token: MONEYBALL - type: word - start_offset: 0 - end_offset: 9 - position: 0 - - synopsis: Apply a character filter. - path: /_analyze - method: GET - request: - payload: - tokenizer: keyword - filter: - - lowercase - char_filter: - - html_strip - text: Moneyball - response: - status: 200 - payload: - tokens: - - token: moneyball - type: word - start_offset: 3 - end_offset: 16 - position: 0 - - synopsis: Combine a lowercase translation with a stop filter. - path: /_analyze - method: GET - request: - payload: - tokenizer: whitespace - filter: - - lowercase - - type: stop - stopwords: - - in - - to - text: Moneyball directed by Bennett Miller - response: - status: 200 - payload: - tokens: - - token: moneyball - type: word - start_offset: 0 - end_offset: 9 - position: 0 diff --git a/tests/default/_core/analyze/filter/asciifolding.yaml b/tests/default/_core/analyze/filter/asciifolding.yaml new file mode 100644 index 000000000..b2af17dcf --- /dev/null +++ b/tests/default/_core/analyze/filter/asciifolding.yaml @@ -0,0 +1,23 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +version: '>= 2.17' +chapters: + - synopsis: Apply a asciifolding filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - asciifolding + text: Léon + response: + status: 200 + payload: + tokens: + - token: Leon + type: word + start_offset: 0 + end_offset: 4 + position: 0 \ No newline at end of file diff --git a/tests/default/_core/analyze/filter/lowercase.yaml b/tests/default/_core/analyze/filter/lowercase.yaml new file mode 100644 index 000000000..cf80ae09b --- /dev/null +++ b/tests/default/_core/analyze/filter/lowercase.yaml @@ -0,0 +1,24 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +chapters: + - synopsis: Apply a lowercase character filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - lowercase + char_filter: + - html_strip + text: Moneyball + response: + status: 200 + payload: + tokens: + - token: moneyball + type: word + start_offset: 3 + end_offset: 16 + position: 0 \ No newline at end of file diff --git a/tests/default/_core/analyze/filter/persian_stem.yaml b/tests/default/_core/analyze/filter/persian_stem.yaml new file mode 100644 index 000000000..973ecbaae --- /dev/null +++ b/tests/default/_core/analyze/filter/persian_stem.yaml @@ -0,0 +1,23 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +version: '>= 2.17' +chapters: + - synopsis: Apply a persian_stem filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - persian_stem + text: جامدات + response: + status: 200 + payload: + tokens: + - token: جامد + type: word + start_offset: 0 + end_offset: 6 + position: 0 \ No newline at end of file diff --git a/tests/default/_core/analyze/filter/porterstem.yaml b/tests/default/_core/analyze/filter/porterstem.yaml new file mode 100644 index 000000000..d7b9d4a06 --- /dev/null +++ b/tests/default/_core/analyze/filter/porterstem.yaml @@ -0,0 +1,23 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +version: '>= 2.17' +chapters: + - synopsis: Apply a porter_stem filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - porter_stem + text: Directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: Directed by Bennett Mil + type: word + start_offset: 0 + end_offset: 26 + position: 0 \ No newline at end of file diff --git a/tests/default/_core/analyze/filter/stop.yaml b/tests/default/_core/analyze/filter/stop.yaml new file mode 100644 index 000000000..bdcf78480 --- /dev/null +++ b/tests/default/_core/analyze/filter/stop.yaml @@ -0,0 +1,26 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +chapters: + - synopsis: Combine a lowercase translation with a stop filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: whitespace + filter: + - lowercase + - type: stop + stopwords: + - in + - to + text: Moneyball directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: moneyball + type: word + start_offset: 0 + end_offset: 9 + position: 0 diff --git a/tests/default/_core/analyze/filter/uppercase.yaml b/tests/default/_core/analyze/filter/uppercase.yaml new file mode 100644 index 000000000..e689916ff --- /dev/null +++ b/tests/default/_core/analyze/filter/uppercase.yaml @@ -0,0 +1,23 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze with a filter. +chapters: + - synopsis: Apply an uppercase character filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - uppercase + text: Moneyball + response: + status: 200 + payload: + tokens: + - token: MONEYBALL + type: word + start_offset: 0 + end_offset: 9 + position: 0 + From 939a292753eeb1b3bffdf5ed4f3039ab5edd2f04 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Tue, 1 Oct 2024 14:41:15 -0500 Subject: [PATCH 33/73] Update bulk descriptions. (#595) Signed-off-by: Archer --- spec/schemas/_core.bulk.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/spec/schemas/_core.bulk.yaml b/spec/schemas/_core.bulk.yaml index 417d1ffd4..5db0782c8 100644 --- a/spec/schemas/_core.bulk.yaml +++ b/spec/schemas/_core.bulk.yaml @@ -29,7 +29,6 @@ components: dynamic_templates: description: |- A map from the full name of fields to the name of dynamic templates. - Defaults to an empty map. If a name matches a dynamic template, then that template will be applied regardless of other match predicates defined in the template. If a field is already defined in the mapping, then this parameter won't be used. type: object @@ -37,12 +36,12 @@ components: type: string pipeline: description: |- - ID of the pipeline to use to preprocess incoming documents. - If the index has a default ingest pipeline specified, then setting the value to `_none` disables the default ingest pipeline for this request. - If a final pipeline is configured it will always run, regardless of the value of this parameter. + The pipeline ID for preprocessing documents. + When the index has a default ingest pipeline specified, then setting the value to `_none` disables the default ingest pipeline for this request. + When a final pipeline is configured, that pipeline will always run, regardless of the value of this parameter. type: string require_alias: - description: If `true`, the request's actions must target an index alias. + description: When `true`, require that all actions target an index alias rather than an index. Default is `false`. type: boolean OperationBase: type: object @@ -69,7 +68,7 @@ components: - type: object properties: require_alias: - description: If `true`, the request's actions must target an index alias. + description: When `true`, the request's actions must target an index alias. type: boolean retry_on_conflict: type: number @@ -80,19 +79,19 @@ components: properties: detect_noop: description: |- - Set to false to disable setting 'result' in the response + When `false` disables the setting 'result' in the response to 'noop' if no change to the document occurred. type: boolean doc: description: A partial update to an existing document. type: object doc_as_upsert: - description: Set to true to use the contents of 'doc' as the value of 'upsert' + description: When `true`, uses the contents of 'doc' as the value of 'upsert'. type: boolean script: $ref: '_common.yaml#/components/schemas/Script' scripted_upsert: - description: Set to true to execute the script whether or not the document exists. + description: When `true`, executes the script whether or not the document exists. type: boolean _source: $ref: '_core.search.yaml#/components/schemas/SourceConfig' @@ -113,8 +112,8 @@ components: type: ['null', string] _index: description: |- - Name of the index associated with the operation. - If the operation targeted a data stream, this is the backing index into which the document was written. + The name of the index associated with the operation. + If the operation targets a data stream, this is the backing index into which the document was written. type: string status: description: HTTP status code returned for the operation. From 62da9f4f3b5f508f41c000e49d1bbea6c2ebda58 Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Wed, 2 Oct 2024 04:46:08 -0700 Subject: [PATCH 34/73] add accurate numeric type in DSL (#597) * add accurate numeric type in DSL Signed-off-by: amberzsy * add CHANGELOG Signed-off-by: amberzsy --------- Signed-off-by: amberzsy --- CHANGELOG.md | 1 + spec/schemas/_common.query_dsl.yaml | 57 +++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fb4ed38f..0a8a9f7ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,6 +169,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) - Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) - Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) +- Fixed inaccurate numeric type in DSL ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597)) ### Security diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 2a23d5ccf..f4683b503 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -280,6 +280,7 @@ components: A boost value between 0 and 1.0 decreases the relevance score. A value greater than 1.0 increases the relevance score. type: number + format: float _name: type: string BoostingQuery: @@ -290,6 +291,7 @@ components: negative_boost: description: Floating point number between 0 and 1.0 used to decrease the relevance scores of documents matching the `negative` query. type: number + format: float negative: $ref: '#/components/schemas/QueryContainer' positive: @@ -307,6 +309,7 @@ components: type: string cutoff_frequency: type: number + format: float high_freq_operator: $ref: '#/components/schemas/Operator' low_freq_operator: @@ -379,6 +382,7 @@ components: tie_breaker: description: Floating point number between 0 and 1.0 used to increase the relevance scores of documents matching multiple query clauses. type: number + format: float required: - queries DistanceFeatureQuery: @@ -447,9 +451,11 @@ components: max_boost: description: Restricts the new score to not exceed the provided limit. type: number + format: float min_score: description: Excludes documents that do not meet the provided score threshold. type: number + format: float query: $ref: '#/components/schemas/QueryContainer' score_mode: @@ -471,6 +477,7 @@ components: $ref: '#/components/schemas/QueryContainer' weight: type: number + format: float - type: object properties: exp: @@ -524,11 +531,13 @@ components: factor: description: Optional factor to multiply the field value with. type: number + format: float missing: description: |- Value used if the document doesn't have that field. The modifier and factor are still applied to it as though it were read from the document. type: number + format: double modifier: $ref: '#/components/schemas/FieldValueFactorModifier' required: @@ -554,6 +563,7 @@ components: seed: oneOf: - type: number + format: int32 - type: string ScriptScoreFunction: type: object @@ -579,9 +589,11 @@ components: max_expansions: description: Maximum number of variations created. type: number + format: int32 prefix_length: description: Number of beginning characters left unchanged when creating expansions. type: number + format: int32 rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' transpositions: @@ -696,11 +708,13 @@ components: Maximum number of child documents that match the query allowed for a returned parent document. If the parent document exceeds this limit, it is excluded from the search results. type: number + format: int32 min_children: description: |- Minimum number of child documents that match the query required to match the query for a returned parent document. If the parent document does not meet this limit, it is excluded from the search results. type: number + format: int32 query: $ref: '#/components/schemas/QueryContainer' score_mode: @@ -791,6 +805,7 @@ components: Maximum number of positions between the matching terms. Intervals produced by the rules further apart than this are not considered matches. type: number + format: int32 ordered: description: If `true`, intervals produced by the rules should appear in the order in which they are specified. type: boolean @@ -861,6 +876,7 @@ components: prefix_length: description: Number of beginning characters left unchanged when creating expansions. type: number + format: int32 term: description: The term to match. type: string @@ -882,6 +898,7 @@ components: Maximum number of positions between the matching terms. Terms further apart than this are not considered matches. type: number + format: int32 ordered: description: If `true`, matching terms must appear in their specified order. type: boolean @@ -942,6 +959,7 @@ components: cutoff_frequency: deprecated: true type: number + format: float fuzziness: $ref: '_common.yaml#/components/schemas/Fuzziness' fuzzy_rewrite: @@ -955,6 +973,7 @@ components: max_expansions: description: Maximum number of terms to which the query will expand. type: number + format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' operator: @@ -962,6 +981,7 @@ components: prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. type: number + format: int32 query: description: Text, number, boolean value or date you wish to find in the provided field. oneOf: @@ -1001,6 +1021,7 @@ components: Maximum number of terms to which the query will expand. Can be applied to the term subqueries constructed for all terms but the final term. type: number + format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' operator: @@ -1010,6 +1031,7 @@ components: Number of beginning characters left unchanged for fuzzy matching. Can be applied to the term subqueries constructed for all terms but the final term. type: number + format: int32 query: description: |- Terms you wish to find in the provided field. @@ -1035,6 +1057,7 @@ components: slop: description: Maximum number of positions allowed between matching tokens. type: number + format: int32 zero_terms_query: $ref: '#/components/schemas/ZeroTermsQuery' required: @@ -1050,12 +1073,14 @@ components: max_expansions: description: Maximum number of terms to which the last provided term of the query value will expand. type: number + format: int32 query: description: Text you wish to find in the provided field. type: string slop: description: Maximum number of positions allowed between matching tokens. type: number + format: int32 zero_terms_query: $ref: '#/components/schemas/ZeroTermsQuery' required: @@ -1076,6 +1101,7 @@ components: This sets the boost factor to use when using this feature. Defaults to deactivated (0). type: number + format: float fail_on_unsupported_field: description: Controls whether the query should fail (throw an exception) if any of the specified fields are not of the supported types (`text` or `keyword`). type: boolean @@ -1099,25 +1125,31 @@ components: max_doc_freq: description: The maximum document frequency above which the terms are ignored from the input document. type: number + format: int32 max_query_terms: description: The maximum number of query terms that can be selected. type: number + format: int32 max_word_length: description: |- The maximum word length above which the terms are ignored. Defaults to unbounded (`0`). type: number + format: int32 min_doc_freq: description: The minimum document frequency below which the terms are ignored from the input document. type: number + format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' min_term_freq: description: The minimum term frequency below which the terms are ignored from the input document. type: number + format: int32 min_word_length: description: The minimum word length below which the terms are ignored. type: number + format: int32 per_field_analyzer: description: Overrides the default analyzer. type: object @@ -1183,6 +1215,7 @@ components: cutoff_frequency: deprecated: true type: number + format: float fields: $ref: '_common.yaml#/components/schemas/Fields' fuzziness: @@ -1200,6 +1233,7 @@ components: max_expansions: description: Maximum number of terms to which the query will expand. type: number + format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' operator: @@ -1207,15 +1241,18 @@ components: prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. type: number + format: int32 query: description: Text, number, boolean value or date you wish to find in the provided field. type: string slop: description: Maximum number of positions allowed between matching tokens. type: number + format: int32 tie_breaker: description: Determines how scores for each per-term blended query and scores across groups are combined. type: number + format: float type: $ref: '#/components/schemas/TextQueryType' zero_terms_query: @@ -1270,8 +1307,10 @@ components: type: integer min_score: type: number + format: float max_distance: type: number + format: float filter: $ref: '#/components/schemas/QueryContainer' ParentIdQuery: @@ -1408,9 +1447,11 @@ components: fuzzy_max_expansions: description: Maximum number of terms to which the query expands for fuzzy matching. type: number + format: int32 fuzzy_prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. type: number + format: int32 fuzzy_rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' fuzzy_transpositions: @@ -1422,11 +1463,13 @@ components: max_determinized_states: description: Maximum number of automaton states required for the query. type: number + format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' phrase_slop: description: Maximum number of positions allowed between matching tokens for phrases. type: number + format: int32 query: description: Query string you wish to parse and use for search. type: string @@ -1445,6 +1488,7 @@ components: tie_breaker: description: How to combine the queries generated from the individual search terms in the resulting `dis_max` query. type: number + format: float time_zone: $ref: '_common.yaml#/components/schemas/TimeZone' type: @@ -1539,6 +1583,7 @@ components: pivot: description: Configurable pivot value so that the result will be less than 0.5. type: number + format: float RankFeatureFunction: type: object RankFeatureFunctionLogarithm: @@ -1549,6 +1594,7 @@ components: scaling_factor: description: Configurable scaling factor. type: number + format: float required: - scaling_factor RankFeatureFunctionLinear: @@ -1563,9 +1609,11 @@ components: pivot: description: Configurable pivot value so that the result will be less than 0.5. type: number + format: float exponent: description: Configurable Exponent. type: number + format: float required: - exponent - pivot @@ -1585,6 +1633,7 @@ components: max_determinized_states: description: Maximum number of automaton states required for the query. type: number + format: int32 rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' value: @@ -1624,6 +1673,7 @@ components: min_score: description: Documents with a score lower than this floating point number are excluded from the search results. type: number + format: float query: $ref: '#/components/schemas/QueryContainer' script: @@ -1661,9 +1711,11 @@ components: fuzzy_max_expansions: description: Maximum number of terms to which the query expands for fuzzy matching. type: number + format: int32 fuzzy_prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. type: number + format: int32 fuzzy_transpositions: description: If `true`, edits for fuzzy matching include transpositions of two adjacent characters (for example, `ab` to `ba`). type: boolean @@ -1762,6 +1814,7 @@ components: end: description: Controls the maximum end position permitted in a match. type: number + format: int32 match: $ref: '#/components/schemas/SpanQuery' required: @@ -1799,6 +1852,7 @@ components: slop: description: Controls the maximum number of intervening unmatched positions permitted. type: number + format: int32 required: - clauses SpanNotQuery: @@ -1811,6 +1865,7 @@ components: The number of tokens from within the include span that can't have overlap with the exclude span. Equivalent to setting both `pre` and `post`. type: number + format: int32 exclude: $ref: '#/components/schemas/SpanQuery' include: @@ -1818,9 +1873,11 @@ components: post: description: The number of tokens after the include span that can't have overlap with the exclude span. type: number + format: int32 pre: description: The number of tokens before the include span that can't have overlap with the exclude span. type: number + format: int32 required: - exclude - include From eb6b844258c7af01faa910183d844143f9019034 Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:00:19 -0700 Subject: [PATCH 35/73] update with more accurate numeric type in _core.yaml (#598) * update with more accurate numeric type in _core.yaml Signed-off-by: amberzsy * update changelog Signed-off-by: amberzsy --------- Signed-off-by: amberzsy --- CHANGELOG.md | 2 +- spec/namespaces/_core.yaml | 51 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a8a9f7ad..e9450972f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -169,7 +169,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) - Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) - Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) -- Fixed inaccurate numeric type in DSL ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597)) +- Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598)) ### Security diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index a9e7b8b0d..09acafcdd 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2306,6 +2306,7 @@ components: max_docs: description: The maximum number of documents to delete. type: number + format: int32 query: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' slice: @@ -2446,10 +2447,12 @@ components: max_docs: description: The maximum number of documents to reindex. type: number + format: int32 script: $ref: '../schemas/_common.yaml#/components/schemas/Script' size: type: number + format: int32 source: $ref: '../schemas/_core.reindex.yaml#/components/schemas/Source' required: @@ -2536,6 +2539,7 @@ components: By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. type: number + format: int32 highlight: $ref: '../schemas/_core.search.yaml#/components/schemas/Highlight' track_total_hits: @@ -2547,6 +2551,7 @@ components: type: object additionalProperties: type: number + format: float docvalue_fields: description: |- Array of wildcard (`*`) patterns. @@ -2561,6 +2566,7 @@ components: Minimum `_score` for matching documents. Documents with a lower `_score` are not included in the search results. type: number + format: float post_filter: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' profile: @@ -2590,6 +2596,7 @@ components: By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. type: number + format: int32 slice: $ref: '../schemas/_common.yaml#/components/schemas/SlicedScroll' sort: @@ -2616,6 +2623,7 @@ components: Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. If set to `0` (default), the query does not terminate early. type: number + format: int32 timeout: description: |- Specifies the period of time to wait for a response from each shard. @@ -2733,6 +2741,7 @@ components: max_docs: description: The maximum number of documents to update. type: number + format: int32 query: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' script: @@ -2761,8 +2770,10 @@ components: maxProperties: 1 took: type: number + format: int64 ingest_took: type: number + format: int64 required: - errors - items @@ -2785,8 +2796,10 @@ components: maxProperties: 1 took: type: number + format: int64 ingest_took: type: number + format: int64 required: - errors - items @@ -2801,6 +2814,7 @@ components: type: boolean num_freed: type: number + format: int32 required: - num_freed - succeeded @@ -2814,6 +2828,7 @@ components: type: boolean count: type: number + format: int64 _shards: $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' required: @@ -3072,6 +3087,7 @@ components: metric_score: description: The overall evaluation quality calculated by the defined metric type: number + format: double details: description: The details section contains one entry for every query in the original requests section, keyed by the search request id type: object @@ -3181,6 +3197,7 @@ components: properties: took: type: number + format: int64 timed_out: type: boolean _shards: @@ -3199,8 +3216,10 @@ components: type: object max_score: type: number + format: float num_reduce_phases: type: number + format: int32 profile: $ref: '../schemas/_core.search.yaml#/components/schemas/Profile' pit_id: @@ -3238,6 +3257,7 @@ components: $ref: '../schemas/_core.termvectors.yaml#/components/schemas/TermVector' took: type: number + format: int64 _version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' required: @@ -3570,6 +3590,7 @@ components: description: Sets the minimum `_score` value that documents must have to be included in the result. schema: type: number + format: float style: form count::query.preference: in: query @@ -3604,6 +3625,7 @@ components: OpenSearch collects documents before sorting. schema: type: number + format: int32 style: form create::path.id: in: path @@ -3750,6 +3772,7 @@ components: description: Only perform the operation if the document has this primary term. schema: type: number + format: int64 style: form delete::query.if_seq_no: in: query @@ -3913,6 +3936,7 @@ components: description: Starting offset. schema: type: number + format: int32 default: 0 style: form delete_by_query::query.ignore_unavailable: @@ -3937,6 +3961,7 @@ components: Defaults to all documents. schema: type: number + format: int32 style: form delete_by_query::query.preference: in: query @@ -3977,6 +4002,7 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number + format: float default: 0 style: form delete_by_query::query.routing: @@ -3999,6 +4025,7 @@ components: description: Size of the scroll request that powers the operation. schema: type: number + format: int32 default: 100 style: form delete_by_query::query.search_timeout: @@ -4066,6 +4093,7 @@ components: Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. schema: type: number + format: int32 style: form delete_by_query::query.timeout: in: query @@ -4114,6 +4142,7 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number + format: float style: form delete_script::path.id: in: path @@ -4706,6 +4735,7 @@ components: description: Only perform the operation if the document has this primary term. schema: type: number + format: int64 style: form index::query.if_seq_no: in: query @@ -4886,6 +4916,7 @@ components: description: Maximum number of concurrent searches the multi search API can execute. schema: type: number + format: int32 style: form msearch::query.max_concurrent_shard_requests: in: query @@ -4893,6 +4924,7 @@ components: description: Maximum number of concurrent shard requests that each sub-search request executes per node. schema: type: number + format: int32 default: 5 style: form msearch::query.pre_filter_shard_size: @@ -4901,6 +4933,7 @@ components: description: Defines a threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method i.e., if date filters are mandatory to match but the shard bounds and the query are disjoint. schema: type: number + format: int32 style: form msearch::query.rest_total_hits_as_int: in: query @@ -4949,6 +4982,7 @@ components: description: Maximum number of concurrent searches the API can run. schema: type: number + format: int32 style: form msearch_template::query.rest_total_hits_as_int: in: query @@ -5198,6 +5232,7 @@ components: Defaults to no throttle. schema: type: number + format: float default: 0 style: form reindex::query.scroll: @@ -5257,6 +5292,7 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number + format: float style: form render_search_template::path.id: in: path @@ -5390,6 +5426,7 @@ components: This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large. schema: type: number + format: int32 default: 512 style: form search::query.ccs_minimize_roundtrips: @@ -5452,6 +5489,7 @@ components: To page through more hits, use the `search_after` parameter. schema: type: number + format: int32 default: 0 style: form search::query.ignore_throttled: @@ -5493,6 +5531,7 @@ components: This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests. schema: type: number + format: int32 default: 5 style: form search::query.pre_filter_shard_size: @@ -5507,6 +5546,7 @@ components: the primary sort of the query targets an indexed field. schema: type: number + format: int32 style: form search::query.preference: in: query @@ -5606,6 +5646,7 @@ components: To page through more hits, use the `search_after` parameter. schema: type: number + format: int32 default: 10 style: form search::query.sort: @@ -5663,6 +5704,7 @@ components: This parameter can only be used when the `suggest_field` and `suggest_text` query string parameters are specified. schema: type: number + format: int32 style: form search::query.suggest_text: in: query @@ -5687,6 +5729,7 @@ components: If set to `0` (default), the query does not terminate early. schema: type: number + format: int32 style: form search::query.timeout: in: query @@ -6061,6 +6104,7 @@ components: description: Only perform the operation if the document has this primary term. schema: type: number + format: int64 style: form update::query.if_seq_no: in: query @@ -6101,6 +6145,7 @@ components: description: Specify how many times should the operation be retried when a conflict occurs. schema: type: number + format: int32 default: 0 style: form update::query.routing: @@ -6238,6 +6283,7 @@ components: description: Starting offset. schema: type: number + format: int32 default: 0 style: form update_by_query::query.ignore_unavailable: @@ -6262,6 +6308,7 @@ components: Defaults to all documents. schema: type: number + format: int32 style: form update_by_query::query.pipeline: in: query @@ -6310,6 +6357,7 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number + format: float default: 0 style: form update_by_query::query.routing: @@ -6332,6 +6380,7 @@ components: description: Size of the scroll request that powers the operation. schema: type: number + format: int32 default: 100 style: form update_by_query::query.search_timeout: @@ -6395,6 +6444,7 @@ components: Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. schema: type: number + format: int32 style: form update_by_query::query.timeout: in: query @@ -6443,4 +6493,5 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number + format: float style: form \ No newline at end of file From bc6ae44c22c3665a9a6668495c807432178ac355 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Sat, 5 Oct 2024 01:29:25 +1300 Subject: [PATCH 36/73] Fix mapping and analysis types (#600) * Fix mapping and analysis types Signed-off-by: Thomas Farr * changelog Signed-off-by: Thomas Farr * Add linter for numeric types Signed-off-by: Thomas Farr * Remove unsupported datastream lifecycle types Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- .cspell | 5 + CHANGELOG.md | 6 +- spec/namespaces/_core.yaml | 80 ++--- spec/schemas/_common.analysis.yaml | 120 +++++-- spec/schemas/_common.mapping.yaml | 153 ++++----- spec/schemas/_common.query_dsl.yaml | 73 ++-- spec/schemas/_common.yaml | 66 ++-- spec/schemas/_core.search.yaml | 23 +- spec/schemas/cluster._common.yaml | 2 - spec/schemas/indices._common.yaml | 311 ++++++++---------- spec/schemas/indices.put_index_template.yaml | 2 - tools/src/_utils/index.ts | 3 + tools/src/helpers.ts | 26 +- .../src/linter/InlineObjectSchemaValidator.ts | 55 ---- tools/src/linter/SchemaVisitingValidator.ts | 99 ++++++ tools/src/linter/SpecValidator.ts | 6 +- .../InlineObjectSchemaValidator.test.ts | 4 +- 17 files changed, 560 insertions(+), 474 deletions(-) delete mode 100644 tools/src/linter/InlineObjectSchemaValidator.ts create mode 100644 tools/src/linter/SchemaVisitingValidator.ts diff --git a/.cspell b/.cspell index 1576a9ec4..3d9ace5d3 100644 --- a/.cspell +++ b/.cspell @@ -76,6 +76,7 @@ kstem kuromoji Kuromoji languageset +latlon Léon localstats Lovins @@ -160,6 +161,8 @@ securityconfig slowlog Slowlog slurpfile +smartcn +Smartcn snapshotted softmax Sorani @@ -179,9 +182,11 @@ termvectors tfidf Tfidf Tika +tlbr tokenfilters translog Translog +trbl tubone Undeploys unigrams diff --git a/CHANGELOG.md b/CHANGELOG.md index e9450972f..023c18b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,7 +111,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added better context for non-2XX responses when `--verbose` is used with the tester tool ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359)) - Locked testing for next release of OpenSearch to a specific SHA ([#431](https://github.com/opensearch-project/opensearch-api-specification/pull/431)) - Replaced nullable with `null` type ([#436](https://github.com/opensearch-project/opensearch-api-specification/pull/436)) -- Split test suite ([#472])(https://github.com/opensearch-project/opensearch-api-specification/pull/472) +- Split test suite ([#472](https://github.com/opensearch-project/opensearch-api-specification/pull/472)) - Changed `WriteResponseBase`'s `_primary_term`, `_seq_no` & `_version` to have `int64` format ([#530](https://github.com/opensearch-project/opensearch-api-specification/pull/530)) - Adjusted indices, shards cat API to test against unassigned indices ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Corrected Sort schema for `/_search`'s request body ([#529](https://github.com/opensearch-project/opensearch-api-specification/issues/529)) @@ -130,6 +130,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358)) - Removed `shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) +- Removed unsupported DataStream Lifecycle types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) ### Fixed @@ -169,7 +170,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) - Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) - Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) -- Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598)) +- Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598))([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) +- Fixed mapping and analysis types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) ### Security diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 09acafcdd..9c5077ee6 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2305,7 +2305,7 @@ components: properties: max_docs: description: The maximum number of documents to delete. - type: number + type: integer format: int32 query: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' @@ -2446,12 +2446,12 @@ components: $ref: '../schemas/_core.reindex.yaml#/components/schemas/Destination' max_docs: description: The maximum number of documents to reindex. - type: number + type: integer format: int32 script: $ref: '../schemas/_common.yaml#/components/schemas/Script' size: - type: number + type: integer format: int32 source: $ref: '../schemas/_core.reindex.yaml#/components/schemas/Source' @@ -2538,7 +2538,7 @@ components: Needs to be non-negative. By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. - type: number + type: integer format: int32 highlight: $ref: '../schemas/_core.search.yaml#/components/schemas/Highlight' @@ -2595,7 +2595,7 @@ components: The number of hits to return. By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. - type: number + type: integer format: int32 slice: $ref: '../schemas/_common.yaml#/components/schemas/SlicedScroll' @@ -2622,7 +2622,7 @@ components: When possible, let OpenSearch perform early termination automatically. Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. If set to `0` (default), the query does not terminate early. - type: number + type: integer format: int32 timeout: description: |- @@ -2740,7 +2740,7 @@ components: properties: max_docs: description: The maximum number of documents to update. - type: number + type: integer format: int32 query: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' @@ -2769,10 +2769,10 @@ components: minProperties: 1 maxProperties: 1 took: - type: number + type: integer format: int64 ingest_took: - type: number + type: integer format: int64 required: - errors @@ -2795,10 +2795,10 @@ components: minProperties: 1 maxProperties: 1 took: - type: number + type: integer format: int64 ingest_took: - type: number + type: integer format: int64 required: - errors @@ -2813,7 +2813,7 @@ components: succeeded: type: boolean num_freed: - type: number + type: integer format: int32 required: - num_freed @@ -2827,7 +2827,7 @@ components: terminated_early: type: boolean count: - type: number + type: integer format: int64 _shards: $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' @@ -3196,7 +3196,7 @@ components: type: object properties: took: - type: number + type: integer format: int64 timed_out: type: boolean @@ -3218,7 +3218,7 @@ components: type: number format: float num_reduce_phases: - type: number + type: integer format: int32 profile: $ref: '../schemas/_core.search.yaml#/components/schemas/Profile' @@ -3256,7 +3256,7 @@ components: additionalProperties: $ref: '../schemas/_core.termvectors.yaml#/components/schemas/TermVector' took: - type: number + type: integer format: int64 _version: $ref: '../schemas/_common.yaml#/components/schemas/VersionNumber' @@ -3624,7 +3624,7 @@ components: If a query reaches this limit, OpenSearch terminates the query early. OpenSearch collects documents before sorting. schema: - type: number + type: integer format: int32 style: form create::path.id: @@ -3771,7 +3771,7 @@ components: name: if_primary_term description: Only perform the operation if the document has this primary term. schema: - type: number + type: integer format: int64 style: form delete::query.if_seq_no: @@ -3935,7 +3935,7 @@ components: name: from description: Starting offset. schema: - type: number + type: integer format: int32 default: 0 style: form @@ -3960,7 +3960,7 @@ components: Maximum number of documents to process. Defaults to all documents. schema: - type: number + type: integer format: int32 style: form delete_by_query::query.preference: @@ -4024,7 +4024,7 @@ components: name: scroll_size description: Size of the scroll request that powers the operation. schema: - type: number + type: integer format: int32 default: 100 style: form @@ -4092,7 +4092,7 @@ components: When possible, let OpenSearch perform early termination automatically. Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. schema: - type: number + type: integer format: int32 style: form delete_by_query::query.timeout: @@ -4734,7 +4734,7 @@ components: name: if_primary_term description: Only perform the operation if the document has this primary term. schema: - type: number + type: integer format: int64 style: form index::query.if_seq_no: @@ -4915,7 +4915,7 @@ components: name: max_concurrent_searches description: Maximum number of concurrent searches the multi search API can execute. schema: - type: number + type: integer format: int32 style: form msearch::query.max_concurrent_shard_requests: @@ -4923,7 +4923,7 @@ components: name: max_concurrent_shard_requests description: Maximum number of concurrent shard requests that each sub-search request executes per node. schema: - type: number + type: integer format: int32 default: 5 style: form @@ -4932,7 +4932,7 @@ components: name: pre_filter_shard_size description: Defines a threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on its rewrite method i.e., if date filters are mandatory to match but the shard bounds and the query are disjoint. schema: - type: number + type: integer format: int32 style: form msearch::query.rest_total_hits_as_int: @@ -4981,7 +4981,7 @@ components: name: max_concurrent_searches description: Maximum number of concurrent searches the API can run. schema: - type: number + type: integer format: int32 style: form msearch_template::query.rest_total_hits_as_int: @@ -5425,7 +5425,7 @@ components: The number of shard results that should be reduced at once on the coordinating node. This value should be used as a protection mechanism to reduce the memory overhead per search request if the potential number of shards in the request can be large. schema: - type: number + type: integer format: int32 default: 512 style: form @@ -5488,7 +5488,7 @@ components: By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. schema: - type: number + type: integer format: int32 default: 0 style: form @@ -5530,7 +5530,7 @@ components: Defines the number of concurrent shard requests per node this search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests. schema: - type: number + type: integer format: int32 default: 5 style: form @@ -5545,7 +5545,7 @@ components: the request targets one or more read-only index; the primary sort of the query targets an indexed field. schema: - type: number + type: integer format: int32 style: form search::query.preference: @@ -5645,7 +5645,7 @@ components: By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. To page through more hits, use the `search_after` parameter. schema: - type: number + type: integer format: int32 default: 10 style: form @@ -5703,7 +5703,7 @@ components: Number of suggestions to return. This parameter can only be used when the `suggest_field` and `suggest_text` query string parameters are specified. schema: - type: number + type: integer format: int32 style: form search::query.suggest_text: @@ -5728,7 +5728,7 @@ components: Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. If set to `0` (default), the query does not terminate early. schema: - type: number + type: integer format: int32 style: form search::query.timeout: @@ -6103,7 +6103,7 @@ components: name: if_primary_term description: Only perform the operation if the document has this primary term. schema: - type: number + type: integer format: int64 style: form update::query.if_seq_no: @@ -6144,7 +6144,7 @@ components: name: retry_on_conflict description: Specify how many times should the operation be retried when a conflict occurs. schema: - type: number + type: integer format: int32 default: 0 style: form @@ -6282,7 +6282,7 @@ components: name: from description: Starting offset. schema: - type: number + type: integer format: int32 default: 0 style: form @@ -6307,7 +6307,7 @@ components: Maximum number of documents to process. Defaults to all documents. schema: - type: number + type: integer format: int32 style: form update_by_query::query.pipeline: @@ -6379,7 +6379,7 @@ components: name: scroll_size description: Size of the scroll request that powers the operation. schema: - type: number + type: integer format: int32 default: 100 style: form @@ -6443,7 +6443,7 @@ components: When possible, let OpenSearch perform early termination automatically. Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. schema: - type: number + type: integer format: int32 style: form update_by_query::query.timeout: diff --git a/spec/schemas/_common.analysis.yaml b/spec/schemas/_common.analysis.yaml index f91d6b4b1..9cae913b6 100644 --- a/spec/schemas/_common.analysis.yaml +++ b/spec/schemas/_common.analysis.yaml @@ -25,6 +25,8 @@ components: - $ref: '#/components/schemas/KuromojiAnalyzer' - $ref: '#/components/schemas/SnowballAnalyzer' - $ref: '#/components/schemas/DutchAnalyzer' + - $ref: '#/components/schemas/SmartcnAnalyzer' + - $ref: '#/components/schemas/CjkAnalyzer' CustomAnalyzer: type: object properties: @@ -41,9 +43,9 @@ components: items: type: string position_increment_gap: - type: number + type: integer position_offset_gap: - type: number + type: integer tokenizer: type: string required: @@ -59,7 +61,7 @@ components: version: $ref: '_common.yaml#/components/schemas/VersionString' max_output_size: - type: number + type: integer preserve_original: type: boolean separator: @@ -218,7 +220,7 @@ components: enum: - standard max_token_length: - type: number + type: integer stopwords: $ref: '#/components/schemas/StopWords' required: @@ -347,10 +349,30 @@ components: $ref: '#/components/schemas/StopWords' required: - type + SmartcnAnalyzer: + type: object + properties: + type: + type: string + enum: + - smartcn + CjkAnalyzer: + type: object + properties: + type: + type: string + enum: + - cjk + stopwords: + $ref: '#/components/schemas/StopWords' + stopwords_path: + type: string CharFilter: oneOf: - - type: string - - $ref: '#/components/schemas/CharFilterDefinition' + - title: name + type: string + - title: definition + $ref: '#/components/schemas/CharFilterDefinition' CharFilterDefinition: type: object discriminator: @@ -446,8 +468,10 @@ components: - type TokenFilter: oneOf: - - type: string - - $ref: '#/components/schemas/TokenFilterDefinition' + - title: name + type: string + - title: definition + $ref: '#/components/schemas/TokenFilterDefinition' TokenFilterDefinition: type: object discriminator: @@ -502,6 +526,7 @@ components: - $ref: '#/components/schemas/IcuTransformTokenFilter' - $ref: '#/components/schemas/PhoneticTokenFilter' - $ref: '#/components/schemas/DictionaryDecompounderTokenFilter' + - $ref: '#/components/schemas/SmartcnStopTokenFilter' AsciiFoldingTokenFilter: allOf: - $ref: '#/components/schemas/TokenFilterBase' @@ -591,9 +616,9 @@ components: enum: - edge_ngram max_gram: - type: number + type: integer min_gram: - type: number + type: integer side: $ref: '#/components/schemas/EdgeNGramSide' preserve_original: @@ -634,7 +659,7 @@ components: enum: - fingerprint max_output_size: - type: number + type: integer separator: type: string required: @@ -678,11 +703,11 @@ components: hyphenation_patterns_path: type: string max_subword_size: - type: number + type: integer min_subword_size: - type: number + type: integer min_word_size: - type: number + type: integer only_longest_match: type: boolean word_list: @@ -774,9 +799,9 @@ components: enum: - length max: - type: number + type: integer min: - type: number + type: integer required: - type LimitTokenCountTokenFilter: @@ -835,9 +860,9 @@ components: enum: - ngram max_gram: - type: number + type: integer min_gram: - type: number + type: integer preserve_original: $ref: '_common.yaml#/components/schemas/Stringifiedboolean' required: @@ -1123,7 +1148,7 @@ components: enum: - truncate length: - type: number + type: integer required: - type UniqueTokenFilter: @@ -1246,7 +1271,7 @@ components: enum: - kuromoji_stemmer minimum_length: - type: number + type: integer required: - minimum_length - type @@ -1347,7 +1372,7 @@ components: IcuCollationDecomposition: type: string enum: - - identical + - canonical - no IcuCollationStrength: type: string @@ -1422,7 +1447,7 @@ components: items: $ref: '#/components/schemas/PhoneticLanguage' max_code_len: - type: number + type: integer name_type: $ref: '#/components/schemas/PhoneticNameType' replace: @@ -1487,6 +1512,17 @@ components: - dictionary_decompounder required: - type + SmartcnStopTokenFilter: + allOf: + - $ref: '#/components/schemas/TokenFilterBase' + - type: object + properties: + type: + type: string + enum: + - smartcn_stop + required: + - type Normalizer: type: object discriminator: @@ -1522,8 +1558,10 @@ components: - type Tokenizer: oneOf: - - type: string - - $ref: '#/components/schemas/TokenizerDefinition' + - title: name + type: string + - title: definition + $ref: '#/components/schemas/TokenizerDefinition' TokenizerDefinition: type: object discriminator: @@ -1543,6 +1581,7 @@ components: - $ref: '#/components/schemas/KuromojiTokenizer' - $ref: '#/components/schemas/PatternTokenizer' - $ref: '#/components/schemas/IcuTokenizer' + - $ref: '#/components/schemas/SmartcnTokenizer' CharGroupTokenizer: allOf: - $ref: '#/components/schemas/TokenizerBase' @@ -1557,7 +1596,7 @@ components: items: type: string max_token_length: - type: number + type: integer required: - tokenize_on_chars - type @@ -1573,9 +1612,9 @@ components: custom_token_chars: type: string max_gram: - type: number + type: integer min_gram: - type: number + type: integer token_chars: type: array items: @@ -1604,7 +1643,7 @@ components: enum: - keyword buffer_size: - type: number + type: integer required: - buffer_size - type @@ -1642,9 +1681,9 @@ components: custom_token_chars: type: string max_gram: - type: number + type: integer min_gram: - type: number + type: integer token_chars: type: array items: @@ -1710,7 +1749,7 @@ components: enum: - standard max_token_length: - type: number + type: integer required: - type UaxEmailUrlTokenizer: @@ -1723,7 +1762,7 @@ components: enum: - uax_url_email max_token_length: - type: number + type: integer required: - type WhitespaceTokenizer: @@ -1736,7 +1775,7 @@ components: enum: - whitespace max_token_length: - type: number + type: integer required: - type KuromojiTokenizer: @@ -1753,7 +1792,7 @@ components: mode: $ref: '#/components/schemas/KuromojiTokenizationMode' nbest_cost: - type: number + type: integer nbest_examples: type: string user_dictionary: @@ -1779,8 +1818,19 @@ components: flags: type: string group: - type: number + type: integer pattern: type: string required: - type + SmartcnTokenizer: + allOf: + - $ref: '#/components/schemas/TokenizerBase' + - type: object + properties: + type: + type: string + enum: + - smartcn_tokenizer + required: + - type diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 2c26487fb..c08585de3 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -163,7 +163,6 @@ components: oneOf: - $ref: '#/components/schemas/BinaryProperty' - $ref: '#/components/schemas/BooleanProperty' - - $ref: '#/components/schemas/DynamicProperty' - $ref: '#/components/schemas/JoinProperty' - $ref: '#/components/schemas/KeywordProperty' - $ref: '#/components/schemas/MatchOnlyTextProperty' @@ -251,7 +250,7 @@ components: additionalProperties: $ref: '#/components/schemas/Property' ignore_above: - type: number + type: integer dynamic: $ref: '#/components/schemas/DynamicMapping' fields: @@ -265,6 +264,7 @@ components: properties: boost: type: number + format: double fielddata: $ref: 'indices._common.yaml#/components/schemas/NumericFielddata' index: @@ -277,61 +277,6 @@ components: - boolean required: - type - DynamicProperty: - allOf: - - $ref: '#/components/schemas/DocValuesPropertyBase' - - type: object - properties: - type: - type: string - enum: - - '{dynamic_property}' - enabled: - type: boolean - null_value: - $ref: '_common.yaml#/components/schemas/FieldValue' - boost: - type: number - coerce: - type: boolean - script: - $ref: '_common.yaml#/components/schemas/Script' - on_script_error: - $ref: '#/components/schemas/OnScriptError' - ignore_malformed: - type: boolean - time_series_metric: - $ref: '#/components/schemas/TimeSeriesMetricType' - analyzer: - type: string - eager_global_ordinals: - type: boolean - index: - type: boolean - index_options: - $ref: '#/components/schemas/IndexOptions' - index_phrases: - type: boolean - index_prefixes: - $ref: '#/components/schemas/TextIndexPrefixes' - norms: - type: boolean - position_increment_gap: - type: number - search_analyzer: - type: string - search_quote_analyzer: - type: string - term_vector: - $ref: '#/components/schemas/TermVectorOption' - format: - type: string - precision_step: - type: number - locale: - type: string - required: - - type OnScriptError: type: string enum: @@ -356,9 +301,9 @@ components: type: object properties: max_chars: - type: number + type: integer min_chars: - type: number + type: integer required: - max_chars - min_chars @@ -400,6 +345,7 @@ components: properties: boost: type: number + format: double eager_global_ordinals: type: boolean index: @@ -493,7 +439,7 @@ components: index_options: $ref: '#/components/schemas/IndexOptions' max_shingle_size: - type: number + type: integer norms: type: boolean search_analyzer: @@ -517,6 +463,7 @@ components: type: string boost: type: number + format: double eager_global_ordinals: type: boolean fielddata: @@ -534,7 +481,7 @@ components: norms: type: boolean position_increment_gap: - type: number + type: integer search_analyzer: type: string search_quote_analyzer: @@ -578,6 +525,7 @@ components: properties: boost: type: number + format: double format: type: string ignore_malformed: @@ -587,7 +535,7 @@ components: null_value: $ref: '_common.yaml#/components/schemas/DateTime' precision_step: - type: number + type: integer type: type: string enum: @@ -601,6 +549,7 @@ components: properties: boost: type: number + format: double fielddata: $ref: 'indices._common.yaml#/components/schemas/NumericFielddata' format: @@ -612,7 +561,7 @@ components: null_value: $ref: '_common.yaml#/components/schemas/DateTime' precision_step: - type: number + type: integer locale: type: string type: @@ -693,6 +642,7 @@ components: properties: boost: type: number + format: double depth_limit: type: number doc_values: @@ -755,7 +705,7 @@ components: items: $ref: '#/components/schemas/SuggestContext' max_input_length: - type: number + type: integer preserve_position_increments: type: boolean preserve_separators: @@ -789,8 +739,7 @@ components: - $ref: '#/components/schemas/PropertyBase' - type: object properties: - value: - type: object + value: {} type: type: string enum: @@ -830,6 +779,7 @@ components: properties: boost: type: number + format: double index: type: boolean ignore_malformed: @@ -869,10 +819,12 @@ components: type: string boost: type: number + format: double index: type: boolean null_value: type: number + format: double enable_position_increments: type: boolean type: @@ -916,6 +868,7 @@ components: deprecated: true distance_error_pct: type: number + format: double deprecated: true type: type: string @@ -924,10 +877,21 @@ components: required: - type GeoOrientation: - type: string - enum: - - left - - right + oneOf: + - title: left + type: string + enum: + - LEFT + - clockwise + - cw + - left + - title: right + type: string + enum: + - RIGHT + - ccw + - counterclockwise + - right GeoStrategy: type: string enum: @@ -991,6 +955,7 @@ components: properties: boost: type: number + format: double coerce: type: boolean ignore_malformed: @@ -1017,6 +982,7 @@ components: - double null_value: type: number + format: double required: - type FloatNumberProperty: @@ -1030,6 +996,7 @@ components: - float null_value: type: number + format: float required: - type HalfFloatNumberProperty: @@ -1043,6 +1010,7 @@ components: - half_float null_value: type: number + format: float required: - type IntegerNumberProperty: @@ -1055,7 +1023,7 @@ components: enum: - integer null_value: - type: number + type: integer required: - type LongNumberProperty: @@ -1068,7 +1036,8 @@ components: enum: - long null_value: - type: number + type: integer + format: int64 required: - type ScaledFloatNumberProperty: @@ -1082,8 +1051,10 @@ components: - scaled_float null_value: type: number + format: double scaling_factor: type: number + format: double required: - type ShortNumberProperty: @@ -1132,27 +1103,11 @@ components: properties: boost: type: number + format: double coerce: type: boolean index: type: boolean - KnnVectorPropertyBase: - type: object - properties: - dimension: - type: number - space_type: - type: string - data_type: - type: string - mode: - type: string - compression_level: - type: string - method: - $ref: '#/components/schemas/KnnVectorMethod' - required: - - dimension KnnVectorMethod: type: object properties: @@ -1164,8 +1119,7 @@ components: type: string parameters: type: object - additionalProperties: - type: object + additionalProperties: {} required: - name DoubleRangeProperty: @@ -1225,14 +1179,29 @@ components: - type KnnVectorProperty: allOf: - - $ref: '#/components/schemas/KnnVectorPropertyBase' + - $ref: '#/components/schemas/DocValuesPropertyBase' - type: object properties: type: type: string enum: - knn_vector + dimension: + type: integer + space_type: + type: string + data_type: + type: string + mode: + type: string + compression_level: + type: string + method: + $ref: '#/components/schemas/KnnVectorMethod' + model_id: + type: string required: + - dimension - type MatchType: type: string diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index f4683b503..872ecc039 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -562,7 +562,7 @@ components: $ref: '_common.yaml#/components/schemas/Field' seed: oneOf: - - type: number + - type: integer format: int32 - type: string ScriptScoreFunction: @@ -588,11 +588,11 @@ components: properties: max_expansions: description: Maximum number of variations created. - type: number + type: integer format: int32 prefix_length: description: Number of beginning characters left unchanged when creating expansions. - type: number + type: integer format: int32 rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' @@ -707,13 +707,13 @@ components: description: |- Maximum number of child documents that match the query allowed for a returned parent document. If the parent document exceeds this limit, it is excluded from the search results. - type: number + type: integer format: int32 min_children: description: |- Minimum number of child documents that match the query required to match the query for a returned parent document. If the parent document does not meet this limit, it is excluded from the search results. - type: number + type: integer format: int32 query: $ref: '#/components/schemas/QueryContainer' @@ -804,7 +804,7 @@ components: description: |- Maximum number of positions between the matching terms. Intervals produced by the rules further apart than this are not considered matches. - type: number + type: integer format: int32 ordered: description: If `true`, intervals produced by the rules should appear in the order in which they are specified. @@ -875,7 +875,7 @@ components: $ref: '_common.yaml#/components/schemas/Fuzziness' prefix_length: description: Number of beginning characters left unchanged when creating expansions. - type: number + type: integer format: int32 term: description: The term to match. @@ -897,7 +897,7 @@ components: description: |- Maximum number of positions between the matching terms. Terms further apart than this are not considered matches. - type: number + type: integer format: int32 ordered: description: If `true`, matching terms must appear in their specified order. @@ -972,7 +972,7 @@ components: type: boolean max_expansions: description: Maximum number of terms to which the query will expand. - type: number + type: integer format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' @@ -980,7 +980,7 @@ components: $ref: '#/components/schemas/Operator' prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. - type: number + type: integer format: int32 query: description: Text, number, boolean value or date you wish to find in the provided field. @@ -1020,7 +1020,7 @@ components: description: |- Maximum number of terms to which the query will expand. Can be applied to the term subqueries constructed for all terms but the final term. - type: number + type: integer format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' @@ -1030,7 +1030,7 @@ components: description: |- Number of beginning characters left unchanged for fuzzy matching. Can be applied to the term subqueries constructed for all terms but the final term. - type: number + type: integer format: int32 query: description: |- @@ -1056,7 +1056,7 @@ components: type: string slop: description: Maximum number of positions allowed between matching tokens. - type: number + type: integer format: int32 zero_terms_query: $ref: '#/components/schemas/ZeroTermsQuery' @@ -1072,14 +1072,14 @@ components: type: string max_expansions: description: Maximum number of terms to which the last provided term of the query value will expand. - type: number + type: integer format: int32 query: description: Text you wish to find in the provided field. type: string slop: description: Maximum number of positions allowed between matching tokens. - type: number + type: integer format: int32 zero_terms_query: $ref: '#/components/schemas/ZeroTermsQuery' @@ -1124,31 +1124,31 @@ components: $ref: '#/components/schemas/Like' max_doc_freq: description: The maximum document frequency above which the terms are ignored from the input document. - type: number + type: integer format: int32 max_query_terms: description: The maximum number of query terms that can be selected. - type: number + type: integer format: int32 max_word_length: description: |- The maximum word length above which the terms are ignored. Defaults to unbounded (`0`). - type: number + type: integer format: int32 min_doc_freq: description: The minimum document frequency below which the terms are ignored from the input document. - type: number + type: integer format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' min_term_freq: description: The minimum term frequency below which the terms are ignored from the input document. - type: number + type: integer format: int32 min_word_length: description: The minimum word length below which the terms are ignored. - type: number + type: integer format: int32 per_field_analyzer: description: Overrides the default analyzer. @@ -1232,7 +1232,7 @@ components: type: boolean max_expansions: description: Maximum number of terms to which the query will expand. - type: number + type: integer format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' @@ -1240,14 +1240,14 @@ components: $ref: '#/components/schemas/Operator' prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. - type: number + type: integer format: int32 query: description: Text, number, boolean value or date you wish to find in the provided field. type: string slop: description: Maximum number of positions allowed between matching tokens. - type: number + type: integer format: int32 tie_breaker: description: Determines how scores for each per-term blended query and scores across groups are combined. @@ -1446,11 +1446,11 @@ components: $ref: '_common.yaml#/components/schemas/Fuzziness' fuzzy_max_expansions: description: Maximum number of terms to which the query expands for fuzzy matching. - type: number + type: integer format: int32 fuzzy_prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. - type: number + type: integer format: int32 fuzzy_rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' @@ -1462,13 +1462,13 @@ components: type: boolean max_determinized_states: description: Maximum number of automaton states required for the query. - type: number + type: integer format: int32 minimum_should_match: $ref: '_common.yaml#/components/schemas/MinimumShouldMatch' phrase_slop: description: Maximum number of positions allowed between matching tokens for phrases. - type: number + type: integer format: int32 query: description: Query string you wish to parse and use for search. @@ -1632,7 +1632,7 @@ components: type: string max_determinized_states: description: Maximum number of automaton states required for the query. - type: number + type: integer format: int32 rewrite: $ref: '_common.yaml#/components/schemas/MultiTermQueryRewrite' @@ -1710,11 +1710,11 @@ components: $ref: '#/components/schemas/SimpleQueryStringFlags' fuzzy_max_expansions: description: Maximum number of terms to which the query expands for fuzzy matching. - type: number + type: integer format: int32 fuzzy_prefix_length: description: Number of beginning characters left unchanged for fuzzy matching. - type: number + type: integer format: int32 fuzzy_transpositions: description: If `true`, edits for fuzzy matching include transpositions of two adjacent characters (for example, `ab` to `ba`). @@ -1813,7 +1813,7 @@ components: properties: end: description: Controls the maximum end position permitted in a match. - type: number + type: integer format: int32 match: $ref: '#/components/schemas/SpanQuery' @@ -1851,7 +1851,7 @@ components: type: boolean slop: description: Controls the maximum number of intervening unmatched positions permitted. - type: number + type: integer format: int32 required: - clauses @@ -1864,7 +1864,7 @@ components: description: |- The number of tokens from within the include span that can't have overlap with the exclude span. Equivalent to setting both `pre` and `post`. - type: number + type: integer format: int32 exclude: $ref: '#/components/schemas/SpanQuery' @@ -1872,11 +1872,11 @@ components: $ref: '#/components/schemas/SpanQuery' post: description: The number of tokens after the include span that can't have overlap with the exclude span. - type: number + type: integer format: int32 pre: description: The number of tokens before the include span that can't have overlap with the exclude span. - type: number + type: integer format: int32 required: - exclude @@ -1928,6 +1928,7 @@ components: type: boolean required: - value + TermsQueryField: type: object properties: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index c3e9468d3..1a58bf8d7 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -51,21 +51,29 @@ components: - as 2 top_right / bottom_left points - as a WKT bounding box. oneOf: - - $ref: '#/components/schemas/CoordsGeoBounds' - - $ref: '#/components/schemas/TopLeftBottomRightGeoBounds' - - $ref: '#/components/schemas/TopRightBottomLeftGeoBounds' - - $ref: '#/components/schemas/WktGeoBounds' + - title: coords + $ref: '#/components/schemas/CoordsGeoBounds' + - title: tlbr + $ref: '#/components/schemas/TopLeftBottomRightGeoBounds' + - title: trbl + $ref: '#/components/schemas/TopRightBottomLeftGeoBounds' + - title: wkt + $ref: '#/components/schemas/WktGeoBounds' CoordsGeoBounds: type: object properties: top: type: number + format: double bottom: type: number + format: double left: type: number + format: double right: type: number + format: double required: - bottom - left @@ -83,27 +91,34 @@ components: - top_left GeoLocation: description: |- - A latitude/longitude as a 2 dimensional point. It can be represented in various ways: + A latitude/longitude as a 2-dimensional point. It can be represented in various ways: - as a `{lat, long}` object - as a geo hash value - as a `[lon, lat]` array - as a string in `", "` or WKT point formats. oneOf: - - $ref: '#/components/schemas/LatLonGeoLocation' - - $ref: '#/components/schemas/GeoHashLocation' - - type: array + - title: latlon + $ref: '#/components/schemas/LatLonGeoLocation' + - title: geohash + $ref: '#/components/schemas/GeoHashLocation' + - title: coords + type: array items: type: number - - type: string + format: double + - title: text + type: string LatLonGeoLocation: type: object properties: lat: description: Latitude type: number + format: double lon: description: Longitude type: number + format: double required: - lat - lon @@ -141,11 +156,14 @@ components: - as a `[x, y]` array - as a string in `"x, y"` or WKT point formats. oneOf: - - $ref: '#/components/schemas/XyCartesianCoordinates' - - type: array + - title: cartesian + $ref: '#/components/schemas/XyCartesianCoordinates' + - title: coords + type: array items: type: number - - type: string + - title: text + type: string XyCartesianCoordinates: x-version-added: '2.4' type: object @@ -419,8 +437,10 @@ components: type: string Script: oneOf: - - $ref: '#/components/schemas/InlineScript' - - $ref: '#/components/schemas/StoredScriptId' + - title: inline + $ref: '#/components/schemas/InlineScript' + - title: stored + $ref: '#/components/schemas/StoredScriptId' InlineScript: allOf: - $ref: '#/components/schemas/ScriptBase' @@ -499,9 +519,12 @@ components: $ref: '#/components/schemas/SortCombinations' SortCombinations: oneOf: - - $ref: '#/components/schemas/Field' - - $ref: '#/components/schemas/FieldWithOrder' - - $ref: '#/components/schemas/SortOptions' + - title: field + $ref: '#/components/schemas/Field' + - title: field_with_order + $ref: '#/components/schemas/FieldWithOrder' + - title: options + $ref: '#/components/schemas/SortOptions' FieldWithOrder: type: object additionalProperties: @@ -589,7 +612,7 @@ components: filter: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' max_children: - type: number + type: integer nested: $ref: '#/components/schemas/NestedSortValue' path: @@ -783,7 +806,8 @@ components: - $ref: '#/components/schemas/UnitSeconds' UnitSeconds: description: Time unit for seconds. - type: number + type: integer + format: int64 TimeOfDay: description: Time of day, expressed as HH:MM:SS. type: string @@ -846,7 +870,7 @@ components: Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. oneOf: - - type: number + - type: integer - type: string Stringifiedboolean: description: |- @@ -892,7 +916,7 @@ components: - in - ine - p - Normalization: + TermFrequencyNormalization: type: string enum: - h1 diff --git a/spec/schemas/_core.search.yaml b/spec/schemas/_core.search.yaml index 87a2e01ef..d56bb06c8 100644 --- a/spec/schemas/_core.search.yaml +++ b/spec/schemas/_core.search.yaml @@ -589,10 +589,10 @@ components: $ref: '_common.yaml#/components/schemas/Name' size: description: The maximum number of hits to return per `inner_hits`. - type: number + type: integer from: description: Inner hit starting document offset. - type: number + type: integer collapse: $ref: '#/components/schemas/FieldCollapse' docvalue_fields: @@ -637,9 +637,7 @@ components: $ref: '#/components/schemas/InnerHits' max_concurrent_group_searches: description: The number of concurrent requests allowed to retrieve the inner_hits per group - type: number - collapse: - $ref: '#/components/schemas/FieldCollapse' + type: integer required: - field Highlight: @@ -755,6 +753,12 @@ components: tags_schema: $ref: '#/components/schemas/HighlighterTagsSchema' HighlighterType: + oneOf: + - title: builtin + $ref: '#/components/schemas/BuiltinHighlighterType' + - title: custom + type: string + BuiltinHighlighterType: type: string enum: - fvh @@ -782,11 +786,14 @@ components: SourceConfig: description: Defines how to fetch a source. Fetching can be disabled entirely, or the source can be filtered. oneOf: - - type: boolean - - type: array + - title: fetch + type: boolean + - title: includes + type: array items: $ref: '_common.yaml#/components/schemas/Field' - - $ref: '#/components/schemas/SourceFilter' + - title: filter + $ref: '#/components/schemas/SourceFilter' SourceFilter: type: object properties: diff --git a/spec/schemas/cluster._common.yaml b/spec/schemas/cluster._common.yaml index 8acd74b43..b70f69ce7 100644 --- a/spec/schemas/cluster._common.yaml +++ b/spec/schemas/cluster._common.yaml @@ -44,5 +44,3 @@ components: type: object additionalProperties: $ref: 'indices._common.yaml#/components/schemas/AliasDefinition' - lifecycle: - $ref: 'indices._common.yaml#/components/schemas/DataStreamLifecycleWithRollover' diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index 90d6776d6..f0440ec75 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -22,18 +22,20 @@ components: type: string soft_deletes: $ref: '#/components/schemas/SoftDeletes' + soft_deletes.retention_lease.period: + $ref: '_common.yaml#/components/schemas/Duration' sort: $ref: '#/components/schemas/IndexSegmentSort' number_of_shards: oneOf: - - type: number + - type: integer - type: string number_of_replicas: oneOf: - - type: number + - type: integer - type: string number_of_routing_shards: - type: number + type: integer check_on_startup: $ref: '#/components/schemas/IndexCheckOnStartup' codec: @@ -43,43 +45,59 @@ components: load_fixed_bitset_filters_eagerly: type: boolean hidden: - oneOf: - - type: boolean - - type: string + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' auto_expand_replicas: type: string merge: - $ref: '#/components/schemas/Merge' + $ref: '#/components/schemas/IndexSettingsMerge' + merge.scheduler.max_thread_count: + $ref: '_common.yaml#/components/schemas/Stringifiedinteger' search: - $ref: '#/components/schemas/SettingsSearch' + $ref: '#/components/schemas/IndexSettingsSearch' + search.idle.after: + $ref: '_common.yaml#/components/schemas/Duration' refresh_interval: $ref: '_common.yaml#/components/schemas/Duration' max_result_window: - type: number + type: integer max_inner_result_window: - type: number + type: integer max_rescore_window: - type: number + type: integer max_docvalue_fields_search: - type: number + type: integer max_script_fields: - type: number + type: integer max_ngram_diff: - type: number + type: integer max_shingle_diff: - type: number + type: integer blocks: $ref: '#/components/schemas/IndexSettingBlocks' + blocks.read_only: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' + blocks.read_only_allow_delete: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' + blocks.read: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' + blocks.write: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' + blocks.metadata: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' max_refresh_listeners: - type: number + type: integer analyze: - $ref: '#/components/schemas/SettingsAnalyze' + $ref: '#/components/schemas/IndexSettingsAnalyze' + analyze.max_token_count: + $ref: '_common.yaml#/components/schemas/Stringifiedinteger' highlight: - $ref: '#/components/schemas/SettingsHighlight' + $ref: '#/components/schemas/IndexSettingsHighlight' + highlight.max_analyzed_offset: + type: integer max_terms_count: - type: number + type: integer max_regex_length: - type: number + type: integer routing: $ref: '#/components/schemas/IndexRouting' gc_deletes: @@ -90,6 +108,8 @@ components: $ref: '_common.yaml#/components/schemas/PipelineName' lifecycle: $ref: '#/components/schemas/IndexSettingsLifecycle' + lifecycle.name: + $ref: '_common.yaml#/components/schemas/Name' provided_name: $ref: '_common.yaml#/components/schemas/Name' creation_date: @@ -101,25 +121,29 @@ components: version: $ref: '#/components/schemas/IndexVersioning' verified_before_close: - oneOf: - - type: boolean - - type: string + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' format: oneOf: - type: string - type: number max_slices_per_scroll: - type: number + type: integer translog: $ref: '#/components/schemas/Translog' + translog.durability: + $ref: '#/components/schemas/TranslogDurability' + translog.flush_threshold_size: + $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' query_string: - $ref: '#/components/schemas/SettingsQueryString' + $ref: '#/components/schemas/IndexSettingsQueryString' + query_string.lenient: + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' priority: oneOf: - type: number - type: string top_metrics_max_size: - type: number + type: integer analysis: $ref: '#/components/schemas/IndexSettingsAnalysis' settings: @@ -127,18 +151,28 @@ components: time_series: $ref: '#/components/schemas/IndexSettingsTimeSeries' queries: - $ref: '#/components/schemas/Queries' + $ref: '#/components/schemas/IndexSettingsQueries' similarity: - $ref: '#/components/schemas/SettingsSimilarity' + $ref: '#/components/schemas/IndexSettingsSimilarity' mapping: - $ref: '#/components/schemas/MappingLimitSettings' - indexing.slowlog: - $ref: '#/components/schemas/IndexingSlowlogSettings' + $ref: '#/components/schemas/IndexSettingsMapping' + indexing: + $ref: '#/components/schemas/IndexSettingsIndexing' indexing_pressure: $ref: '#/components/schemas/IndexingPressure' store: - $ref: '#/components/schemas/Storage' - additionalProperties: {} + $ref: '#/components/schemas/IndexSettingsStore' + knn: + type: boolean + knn.algo_param.ef_search: + type: integer + additionalProperties: + title: custom_settings + IndexSettingsIndexing: + type: object + properties: + slowlog: + $ref: '#/components/schemas/IndexingSlowlog' SoftDeletes: type: object properties: @@ -198,42 +232,40 @@ components: - checksum - 'false' - 'true' - Merge: + IndexSettingsMerge: type: object properties: scheduler: - $ref: '#/components/schemas/MergeScheduler' - MergeScheduler: + $ref: '#/components/schemas/IndexSettingsMergeScheduler' + IndexSettingsMergeScheduler: type: object properties: max_thread_count: $ref: '_common.yaml#/components/schemas/Stringifiedinteger' max_merge_count: $ref: '_common.yaml#/components/schemas/Stringifiedinteger' - SettingsSearch: + IndexSettingsSearch: type: object properties: idle: $ref: '#/components/schemas/SearchIdle' slowlog: - $ref: '#/components/schemas/SlowlogSettings' + $ref: '#/components/schemas/SearchSlowlog' SearchIdle: type: object properties: after: $ref: '_common.yaml#/components/schemas/Duration' - SlowlogSettings: + SearchSlowlog: type: object properties: level: type: string - source: - type: number reformat: type: boolean threshold: - $ref: '#/components/schemas/SlowlogThresholds' - SlowlogThresholds: + $ref: '#/components/schemas/SearchSlowlogThresholds' + SearchSlowlogThresholds: type: object properties: query: @@ -264,16 +296,16 @@ components: $ref: '_common.yaml#/components/schemas/Stringifiedboolean' metadata: $ref: '_common.yaml#/components/schemas/Stringifiedboolean' - SettingsAnalyze: + IndexSettingsAnalyze: type: object properties: max_token_count: $ref: '_common.yaml#/components/schemas/Stringifiedinteger' - SettingsHighlight: + IndexSettingsHighlight: type: object properties: max_analyzed_offset: - type: number + type: integer IndexRouting: type: object properties: @@ -315,9 +347,7 @@ components: type: object properties: threshold_enabled: - oneOf: - - type: boolean - - type: string + $ref: '_common.yaml#/components/schemas/Stringifiedboolean' IndexRoutingRebalance: type: object properties: @@ -344,7 +374,7 @@ components: If specified, this is the timestamp used to calculate the index age for its phase transitions. Use this setting if you create a new index that contains old data and want to use the original creation date to calculate the index age. Specified as a Unix epoch value in milliseconds. - type: number + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitMillis' parse_origination_date: description: |- Set to true to parse the origination date from the index name. This origination date is used to calculate the index age @@ -388,6 +418,8 @@ components: TranslogDurability: type: string enum: + - ASYNC + - REQUEST - async - request TranslogRetention: @@ -397,13 +429,11 @@ components: $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' age: $ref: '_common.yaml#/components/schemas/Duration' - SettingsQueryString: + IndexSettingsQueryString: type: object properties: lenient: $ref: '_common.yaml#/components/schemas/Stringifiedboolean' - required: - - lenient IndexSettingsAnalysis: type: object properties: @@ -434,44 +464,46 @@ components: $ref: '_common.yaml#/components/schemas/DateTime' start_time: $ref: '_common.yaml#/components/schemas/DateTime' - Queries: + IndexSettingsQueries: type: object properties: cache: - $ref: '#/components/schemas/CacheQueries' - CacheQueries: + $ref: '#/components/schemas/IndexSettingsQueriesCache' + IndexSettingsQueriesCache: type: object properties: enabled: type: boolean required: - enabled - SettingsSimilarity: + IndexSettingsSimilarity: type: object properties: bm25: - $ref: '#/components/schemas/SettingsSimilarityBm25' + $ref: '#/components/schemas/IndexSettingsSimilarityBm25' dfi: - $ref: '#/components/schemas/SettingsSimilarityDfi' + $ref: '#/components/schemas/IndexSettingsSimilarityDfi' dfr: - $ref: '#/components/schemas/SettingsSimilarityDfr' + $ref: '#/components/schemas/IndexSettingsSimilarityDfr' ib: - $ref: '#/components/schemas/SettingsSimilarityIb' + $ref: '#/components/schemas/IndexSettingsSimilarityIb' lmd: - $ref: '#/components/schemas/SettingsSimilarityLmd' + $ref: '#/components/schemas/IndexSettingsSimilarityLmd' lmj: - $ref: '#/components/schemas/SettingsSimilarityLmj' + $ref: '#/components/schemas/IndexSettingsSimilarityLmj' scripted_tfidf: - $ref: '#/components/schemas/SettingsSimilarityScriptedTfidf' - SettingsSimilarityBm25: + $ref: '#/components/schemas/IndexSettingsSimilarityScriptedTfidf' + IndexSettingsSimilarityBm25: type: object properties: b: type: number + format: float discount_overlaps: type: boolean k1: type: number + format: float type: type: string enum: @@ -481,7 +513,7 @@ components: - discount_overlaps - k1 - type - SettingsSimilarityDfi: + IndexSettingsSimilarityDfi: type: object properties: independence_measure: @@ -493,7 +525,7 @@ components: required: - independence_measure - type - SettingsSimilarityDfr: + IndexSettingsSimilarityDfr: type: object properties: after_effect: @@ -501,7 +533,7 @@ components: basic_model: $ref: '_common.yaml#/components/schemas/DFRBasicModel' normalization: - $ref: '_common.yaml#/components/schemas/Normalization' + $ref: '_common.yaml#/components/schemas/TermFrequencyNormalization' type: type: string enum: @@ -511,7 +543,7 @@ components: - basic_model - normalization - type - SettingsSimilarityIb: + IndexSettingsSimilarityIb: type: object properties: distribution: @@ -519,7 +551,7 @@ components: lambda: $ref: '_common.yaml#/components/schemas/IBLambda' normalization: - $ref: '_common.yaml#/components/schemas/Normalization' + $ref: '_common.yaml#/components/schemas/TermFrequencyNormalization' type: type: string enum: @@ -529,11 +561,12 @@ components: - lambda - normalization - type - SettingsSimilarityLmd: + IndexSettingsSimilarityLmd: type: object properties: mu: type: number + format: float type: type: string enum: @@ -541,11 +574,12 @@ components: required: - mu - type - SettingsSimilarityLmj: + IndexSettingsSimilarityLmj: type: object properties: lambda: type: number + format: float type: type: string enum: @@ -553,7 +587,7 @@ components: required: - lambda - type - SettingsSimilarityScriptedTfidf: + IndexSettingsSimilarityScriptedTfidf: type: object properties: script: @@ -565,26 +599,26 @@ components: required: - script - type - MappingLimitSettings: + IndexSettingsMapping: type: object properties: coerce: type: boolean total_fields: - $ref: '#/components/schemas/MappingLimitSettingsTotalFields' + $ref: '#/components/schemas/IndexSettingsMappingLimitTotalFields' depth: - $ref: '#/components/schemas/MappingLimitSettingsDepth' + $ref: '#/components/schemas/IndexSettingsMappingLimitDepth' nested_fields: - $ref: '#/components/schemas/MappingLimitSettingsNestedFields' + $ref: '#/components/schemas/IndexSettingsMappingLimitNestedFields' nested_objects: - $ref: '#/components/schemas/MappingLimitSettingsNestedObjects' + $ref: '#/components/schemas/IndexSettingsMappingLimitNestedObjects' field_name_length: - $ref: '#/components/schemas/MappingLimitSettingsFieldNameLength' + $ref: '#/components/schemas/IndexSettingsMappingLimitFieldNameLength' dimension_fields: - $ref: '#/components/schemas/MappingLimitSettingsDimensionFields' + $ref: '#/components/schemas/IndexSettingsMappingLimitDimensionFields' ignore_malformed: type: boolean - MappingLimitSettingsTotalFields: + IndexSettingsMappingLimitTotalFields: type: object properties: limit: @@ -592,16 +626,18 @@ components: The maximum number of fields in an index. Field and object mappings, as well as field aliases count towards this limit. The limit is in place to prevent mappings and searches from becoming too large. Higher values can lead to performance degradations and memory issues, especially in clusters with a high load or few resources. - type: number - MappingLimitSettingsDepth: + type: integer + format: int64 + IndexSettingsMappingLimitDepth: type: object properties: limit: description: |- The maximum depth for a field, which is measured as the number of inner objects. For instance, if all fields are defined at the root object level, then the depth is 1. If there is one object mapping, then the depth is 2, etc. - type: number - MappingLimitSettingsNestedFields: + type: integer + format: int64 + IndexSettingsMappingLimitNestedFields: type: object properties: limit: @@ -609,16 +645,18 @@ components: The maximum number of distinct nested mappings in an index. The nested type should only be used in special cases, when arrays of objects need to be queried independently of each other. To safeguard against poorly designed mappings, this setting limits the number of unique nested types per index. - type: number - MappingLimitSettingsNestedObjects: + type: integer + format: int64 + IndexSettingsMappingLimitNestedObjects: type: object properties: limit: description: |- The maximum number of nested JSON objects that a single document can contain across all nested types. This limit helps to prevent out of memory errors when a document contains too many nested objects. - type: number - MappingLimitSettingsFieldNameLength: + type: integer + format: int64 + IndexSettingsMappingLimitFieldNameLength: type: object properties: limit: @@ -626,22 +664,24 @@ components: Setting for the maximum length of a field name. This setting isn't really something that addresses mappings explosion but might still be useful if you want to limit the field length. It usually shouldn't be necessary to set this setting. The default is okay unless a user starts to add a huge number of fields with really long names. Default is `Long.MAX_VALUE` (no limit). - type: number - MappingLimitSettingsDimensionFields: + type: integer + format: int64 + IndexSettingsMappingLimitDimensionFields: type: object properties: limit: description: |- [preview] This functionality is in technical preview and may be changed or removed in a future release. OpenSearch will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. - type: number - IndexingSlowlogSettings: + type: integer + format: int64 + IndexingSlowlog: type: object properties: level: type: string source: - type: number + type: integer reformat: type: boolean threshold: @@ -666,8 +706,11 @@ components: Number of outstanding bytes that may be consumed by indexing requests. When this limit is reached or exceeded, the node will reject new coordinating and primary operations. When replica operations consume 1.5x this limit, the node will reject new replica operations. Defaults to 10% of the heap. - type: number - Storage: + oneOf: + - type: string + - type: integer + format: int64 + IndexSettingsStore: type: object properties: type: @@ -705,10 +748,12 @@ components: properties: max: type: number + format: double min: type: number + format: double min_segment_size: - type: number + type: integer required: - max - min @@ -739,65 +784,6 @@ components: If `true`, the alias is hidden. All indices for the alias must have the same `is_hidden` value. type: boolean - DataStreamLifecycleWithRollover: - type: object - properties: - data_retention: - $ref: '_common.yaml#/components/schemas/Duration' - downsampling: - $ref: '#/components/schemas/DataStreamLifecycleDownsampling' - rollover: - $ref: '#/components/schemas/DataStreamLifecycleRolloverConditions' - DataStreamLifecycleDownsampling: - type: object - properties: - rounds: - description: The list of downsampling rounds to execute as part of this downsampling configuration - type: array - items: - $ref: '#/components/schemas/DownsamplingRound' - required: - - rounds - DownsamplingRound: - type: object - properties: - after: - $ref: '_common.yaml#/components/schemas/Duration' - config: - $ref: '#/components/schemas/DownsampleConfig' - required: - - after - - config - DownsampleConfig: - type: object - properties: - fixed_interval: - $ref: '_common.yaml#/components/schemas/DurationLarge' - required: - - fixed_interval - DataStreamLifecycleRolloverConditions: - type: object - properties: - min_age: - $ref: '_common.yaml#/components/schemas/Duration' - max_age: - type: string - min_docs: - type: number - max_docs: - type: number - min_size: - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' - max_size: - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' - min_primary_shard_size: - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' - max_primary_shard_size: - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' - min_primary_shard_docs: - type: number - max_primary_shard_docs: - type: number IndexState: type: object properties: @@ -813,8 +799,6 @@ components: $ref: '#/components/schemas/IndexSettings' data_stream: $ref: '_common.yaml#/components/schemas/DataStreamName' - lifecycle: - $ref: '#/components/schemas/DataStreamLifecycle' Alias: type: object properties: @@ -834,13 +818,6 @@ components: $ref: '_common.yaml#/components/schemas/Routing' search_routing: $ref: '_common.yaml#/components/schemas/Routing' - DataStreamLifecycle: - type: object - properties: - data_retention: - $ref: '_common.yaml#/components/schemas/Duration' - downsampling: - $ref: '#/components/schemas/DataStreamLifecycleDownsampling' DataStream: type: object properties: @@ -869,8 +846,6 @@ components: type: array items: $ref: '#/components/schemas/DataStreamIndex' - lifecycle: - $ref: '#/components/schemas/DataStreamLifecycleWithRollover' name: $ref: '_common.yaml#/components/schemas/DataStreamName' replicated: @@ -970,8 +945,6 @@ components: $ref: '_common.mapping.yaml#/components/schemas/TypeMapping' settings: $ref: '#/components/schemas/IndexSettings' - lifecycle: - $ref: '#/components/schemas/DataStreamLifecycleWithRollover' IndexTemplateDataStreamConfiguration: type: object properties: diff --git a/spec/schemas/indices.put_index_template.yaml b/spec/schemas/indices.put_index_template.yaml index bdee9bbe9..e8383c2b2 100644 --- a/spec/schemas/indices.put_index_template.yaml +++ b/spec/schemas/indices.put_index_template.yaml @@ -22,5 +22,3 @@ components: $ref: '_common.mapping.yaml#/components/schemas/TypeMapping' settings: $ref: 'indices._common.yaml#/components/schemas/IndexSettings' - lifecycle: - $ref: 'indices._common.yaml#/components/schemas/DataStreamLifecycle' diff --git a/tools/src/_utils/index.ts b/tools/src/_utils/index.ts index 5177fd86e..02d9a18aa 100644 --- a/tools/src/_utils/index.ts +++ b/tools/src/_utils/index.ts @@ -15,6 +15,9 @@ import { to_json } from "../helpers"; export const HTTP_METHODS: OpenAPIV3.HttpMethods[] = Object.values(OpenAPIV3.HttpMethods) export type SchemaObjectType = OpenAPIV3.ArraySchemaObjectType | OpenAPIV3.NonArraySchemaObjectType export const SCHEMA_OBJECT_TYPES: SchemaObjectType[] = ['array', 'boolean', 'object', 'number', 'string', 'integer'] +export const SCHEMA_NUMERIC_TYPES: SchemaObjectType[] = ['number', 'integer'] +export const SCHEMA_NUMBER_FORMATS: string[] = ['float', 'double'] +export const SCHEMA_INTEGER_FORMATS: string[] = ['int32', 'int64'] export function is_ref (o: MaybeRef): o is OpenAPIV3.ReferenceObject { return o != null && typeof o === 'object' && '$ref' in o diff --git a/tools/src/helpers.ts b/tools/src/helpers.ts index 4f1b346ef..4144afd31 100644 --- a/tools/src/helpers.ts +++ b/tools/src/helpers.ts @@ -124,13 +124,25 @@ export function read_yaml> (file_path: string, exclude_s } export function write_yaml (file_path: string, content: any): void { - write_text(file_path, YAML.stringify( - content, - { - lineWidth: 0, - singleQuote: true, - aliasDuplicateObjects: false - })) + const doc = new YAML.Document(content, null, { aliasDuplicateObjects: false }) + + YAML.visit(doc, { + // eslint-disable-next-line @typescript-eslint/naming-convention + Scalar(_, node) { + if (typeof node.value === 'string') { + const value = node.value.toLowerCase(); + // Ensure "human" boolean string values are quoted as old YAML parsers might coerce them to boolean true/false + if (value === 'no' || value === 'yes' || value === 'n' || value === 'y' || value === 'off' || value === 'on') { + node.type = 'QUOTE_SINGLE' + } + } + } + }) + + write_text(file_path, doc.toString({ + lineWidth: 0, + singleQuote: true + })) } export function to_json(content: any, replacer?: (this: any, key: string, value: any) => any): string { diff --git a/tools/src/linter/InlineObjectSchemaValidator.ts b/tools/src/linter/InlineObjectSchemaValidator.ts deleted file mode 100644 index 1f3e06134..000000000 --- a/tools/src/linter/InlineObjectSchemaValidator.ts +++ /dev/null @@ -1,55 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import type NamespacesFolder from './components/NamespacesFolder' -import type SchemasFolder from './components/SchemasFolder' -import { type ValidationError } from 'types' -import { SchemaVisitor } from '../_utils/SpecificationVisitor' -import { is_ref, type MaybeRef, SpecificationContext } from '../_utils' -import { type OpenAPIV3 } from 'openapi-types' - -export default class InlineObjectSchemaValidator { - private readonly _namespaces_folder: NamespacesFolder - private readonly _schemas_folder: SchemasFolder - - constructor (namespaces_folder: NamespacesFolder, schemas_folder: SchemasFolder) { - this._namespaces_folder = namespaces_folder - this._schemas_folder = schemas_folder - } - - validate (): ValidationError[] { - const errors: ValidationError[] = [] - - const visitor = new SchemaVisitor((ctx, schema) => { - this.#validate_schema(ctx, schema, errors) - }); - - [ - ...this._namespaces_folder.files, - ...this._schemas_folder.files - ].forEach(f => { visitor.visit_specification(new SpecificationContext(f.file), f.spec()) }) - - return errors - } - - #validate_schema (ctx: SpecificationContext, schema: MaybeRef, errors: ValidationError[]): void { - if (is_ref(schema) || schema.type !== 'object' || schema.properties === undefined) { - return - } - - const ancestry = ctx.keys.reverse() - - if (ancestry[1] === 'properties' || - ancestry[0] === 'additionalProperties' || - ancestry[0] === 'items' || - (ancestry[0] === 'schema' && ancestry[2] === 'parameters' && ancestry[3] !== 'components')) { - errors.push(ctx.error('object schemas should be defined out-of-line via a $ref')) - } - } -} diff --git a/tools/src/linter/SchemaVisitingValidator.ts b/tools/src/linter/SchemaVisitingValidator.ts new file mode 100644 index 000000000..bf3beff82 --- /dev/null +++ b/tools/src/linter/SchemaVisitingValidator.ts @@ -0,0 +1,99 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import type NamespacesFolder from './components/NamespacesFolder' +import type SchemasFolder from './components/SchemasFolder' +import { type ValidationError } from 'types' +import { SchemaVisitor } from '../_utils/SpecificationVisitor' +import { + is_ref, + type MaybeRef, + SCHEMA_INTEGER_FORMATS, + SCHEMA_NUMBER_FORMATS, + SCHEMA_NUMERIC_TYPES, + SpecificationContext +} from '../_utils' +import { type OpenAPIV3 } from 'openapi-types' + +export default class SchemaVisitingValidator { + private readonly _namespaces_folder: NamespacesFolder + private readonly _schemas_folder: SchemasFolder + + constructor (namespaces_folder: NamespacesFolder, schemas_folder: SchemasFolder) { + this._namespaces_folder = namespaces_folder + this._schemas_folder = schemas_folder + } + + validate (): ValidationError[] { + const errors: ValidationError[] = [] + + const validating_functions = [ + this.#validate_inline_object_schema.bind(this), + this.#validate_numeric_schema.bind(this) + ] + + const visitor = new SchemaVisitor((ctx, schema) => { + for (const f of validating_functions) { + f(ctx, schema, errors) + } + }); + + [ + ...this._namespaces_folder.files, + ...this._schemas_folder.files + ].forEach(f => { visitor.visit_specification(new SpecificationContext(f.file), f.spec()) }) + + return errors + } + + #validate_inline_object_schema (ctx: SpecificationContext, schema: MaybeRef, errors: ValidationError[]): void { + if (is_ref(schema) || schema.type !== 'object' || schema.properties === undefined) { + return + } + + const ancestry = ctx.keys.reverse() + + if (ancestry[1] === 'properties' || + ancestry[0] === 'additionalProperties' || + ancestry[0] === 'items' || + (ancestry[0] === 'schema' && ancestry[2] === 'parameters' && ancestry[3] !== 'components')) { + errors.push(ctx.error('object schemas should be defined out-of-line via a $ref')) + } + } + + #validate_numeric_schema (ctx: SpecificationContext, schema: MaybeRef, errors: ValidationError[]): void { + if (is_ref(schema) || schema.type === undefined || !SCHEMA_NUMERIC_TYPES.includes(schema.type)) { + return + } + + if (schema.type === 'number') { + if (schema.format === undefined || SCHEMA_NUMBER_FORMATS.includes(schema.format)) { + return + } + + if (SCHEMA_INTEGER_FORMATS.includes(schema.format)) { + errors.push(ctx.error(`schema of type 'number' with format '${schema.format}' should instead be of type 'integer'`)) + } else { + errors.push(ctx.error(`schema of type 'number' with format '${schema.format}' is invalid, expected one of: ${SCHEMA_NUMBER_FORMATS.join(', ')}`)) + } + } + + if (schema.type === 'integer') { + if (schema.format === undefined || SCHEMA_INTEGER_FORMATS.includes(schema.format)) { + return + } + + if (SCHEMA_NUMBER_FORMATS.includes(schema.format)) { + errors.push(ctx.error(`schema of type 'integer' with format '${schema.format}' should instead be of type 'number'`)) + } else { + errors.push(ctx.error(`schema of type 'integer' with format '${schema.format}' is invalid, expected one of: ${SCHEMA_INTEGER_FORMATS.join(', ')}`)) + } + } + } +} diff --git a/tools/src/linter/SpecValidator.ts b/tools/src/linter/SpecValidator.ts index b954d3acc..462fbe93b 100644 --- a/tools/src/linter/SpecValidator.ts +++ b/tools/src/linter/SpecValidator.ts @@ -13,7 +13,7 @@ import { type ValidationError } from 'types' import SchemaRefsValidator from './SchemaRefsValidator' import SupersededOperationsFile from './components/SupersededOperationsFile' import InfoFile from './components/InfoFile' -import InlineObjectSchemaValidator from './InlineObjectSchemaValidator' +import SchemaVisitingValidator from './SchemaVisitingValidator' import SchemasValidator from './SchemasValidator' import { type Logger } from '../Logger' @@ -25,7 +25,7 @@ export default class SpecValidator { schemas_folder: SchemasFolder schemas_validator: SchemasValidator schema_refs_validator: SchemaRefsValidator - inline_object_schema_validator: InlineObjectSchemaValidator + inline_object_schema_validator: SchemaVisitingValidator constructor (root_folder: string, logger: Logger) { this.logger = logger @@ -35,7 +35,7 @@ export default class SpecValidator { this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`) this.schemas_validator = new SchemasValidator(root_folder, logger) this.schema_refs_validator = new SchemaRefsValidator(this.namespaces_folder, this.schemas_folder) - this.inline_object_schema_validator = new InlineObjectSchemaValidator(this.namespaces_folder, this.schemas_folder) + this.inline_object_schema_validator = new SchemaVisitingValidator(this.namespaces_folder, this.schemas_folder) } validate (): ValidationError[] { diff --git a/tools/tests/linter/InlineObjectSchemaValidator.test.ts b/tools/tests/linter/InlineObjectSchemaValidator.test.ts index 8efeea258..7e3b2514c 100644 --- a/tools/tests/linter/InlineObjectSchemaValidator.test.ts +++ b/tools/tests/linter/InlineObjectSchemaValidator.test.ts @@ -9,13 +9,13 @@ import SchemasFolder from 'linter/components/SchemasFolder' import NamespacesFolder from 'linter/components/NamespacesFolder' -import InlineObjectSchemaValidator from 'linter/InlineObjectSchemaValidator' +import SchemaVisitingValidator from 'linter/SchemaVisitingValidator' test('validate()', () => { const root_folder = './tools/tests/linter/fixtures/inline_object_schema_validator' const namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`) const schemas_folder = new SchemasFolder(`${root_folder}/schemas`) - const validator = new InlineObjectSchemaValidator(namespaces_folder, schemas_folder) + const validator = new SchemaVisitingValidator(namespaces_folder, schemas_folder) expect(validator.validate()).toEqual([ { file: 'namespaces/ops.yaml', From 118be4f187a27d431022ef0269fde53ecc7369fa Mon Sep 17 00:00:00 2001 From: Vatsal <36672090+imvtsl@users.noreply.github.com> Date: Sat, 5 Oct 2024 05:00:38 -0700 Subject: [PATCH 37/73] Add 404 spec for delete index, get document, delete document APIs (#589) * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal * Add 404 spec for delete index, get document, delete document APIs Signed-off-by: Vatsal --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 14 ++++++++++++ spec/namespaces/indices.yaml | 7 ++++++ spec/schemas/_common.yaml | 4 ++++ spec/schemas/indices._common.yaml | 26 +++++++++++++++++++++ tests/default/indices/doc.yaml | 16 +++++++++++++ tests/default/indices/index.yaml | 10 +++++++- tools/src/tester/ChapterReader.ts | 2 +- tools/tests/tester/ChapterReader.test.ts | 29 ++++++++++++++++++++++++ 9 files changed, 107 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 023c18b2f..b323df068 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_ml/agents/_register`, `/_plugins/_ml/connectors/_create`, `DELETE /_plugins/_ml/agents/{agent_id}`, `DELETE /_plugins/_ml/connectors/{connector_id}` ([#228](https://github.com/opensearch-project/opensearch-api-specification/issues/228)) - Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) - Added `persian_stem` filter ([#592](https://github.com/opensearch-project/opensearch-api-specification/pull/592)) +- Added `404` response for `DELETE /{index}`, `GET /{index}/_doc/{id}`, `DELETE /{index}/_doc/{id}` ([#589](https://github.com/opensearch-project/opensearch-api-specification/pull/589)) ### Changed diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9c5077ee6..79d1343e7 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -1368,6 +1368,8 @@ paths: responses: '200': $ref: '#/components/responses/get@200' + '404': + $ref: '#/components/responses/get@404' head: operationId: exists.0 x-operation-group: exists @@ -1468,6 +1470,8 @@ paths: responses: '200': $ref: '#/components/responses/delete@200' + '404': + $ref: '#/components/responses/delete@404' /{index}/_explain/{id}: get: operationId: explain.0 @@ -2857,6 +2861,11 @@ components: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' + delete@404: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' delete_all_pits@200: content: application/json: @@ -2945,6 +2954,11 @@ components: application/json: schema: $ref: '../schemas/_core.get.yaml#/components/schemas/GetResult' + get@404: + content: + application/json: + schema: + $ref: '../schemas/_core.get.yaml#/components/schemas/GetResult' get_all_pits@200: content: application/json: diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index dcd758619..ad4a4c8dd 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -1044,6 +1044,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.delete@200' + '404': + $ref: '#/components/responses/indices.delete@404' /{index}/_alias: get: operationId: indices.get_alias.2 @@ -2400,6 +2402,11 @@ components: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndicesResponseBase' + indices.delete@404: + content: + application/json: + schema: + $ref: '../schemas/indices._common.yaml#/components/schemas/IndexErrorCause' indices.delete_alias@200: content: application/json: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 1a58bf8d7..33b6ea830 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -2265,3 +2265,7 @@ components: BatchSize: type: integer format: int64 + ResourceType: + type: string + enum: + - index_or_alias diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index f0440ec75..2e3763cc4 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -983,3 +983,29 @@ components: - mappings - order - settings + IndexErrorCause: + type: object + properties: + type: + description: The type of error + type: string + reason: + description: A human-readable explanation of the error, in english + type: string + root_cause: + type: array + items: + $ref: '#/components/schemas/IndexErrorCause' + index: + $ref: '_common.yaml#/components/schemas/IndexName' + index_uuid: + $ref: '_common.yaml#/components/schemas/Uuid' + resource.id: + $ref: '_common.yaml#/components/schemas/IndexName' + resource.type: + $ref: '_common.yaml#/components/schemas/ResourceType' + required: + - type + additionalProperties: + title: metadata + description: Additional details about the error. diff --git a/tests/default/indices/doc.yaml b/tests/default/indices/doc.yaml index 6eb5a8a5d..5396a52bf 100644 --- a/tests/default/indices/doc.yaml +++ b/tests/default/indices/doc.yaml @@ -57,3 +57,19 @@ chapters: id: '1' response: status: 200 + - synopsis: Retrieve a nonexistent document. + path: /{index}/_doc/{id} + method: GET + parameters: + index: movies + id: '1' + response: + status: 404 + - synopsis: Delete a nonexistent document. + path: /{index}/_doc/{id} + method: DELETE + parameters: + index: movies + id: '1' + response: + status: 404 diff --git a/tests/default/indices/index.yaml b/tests/default/indices/index.yaml index 4b34c8937..943aa9ca4 100644 --- a/tests/default/indices/index.yaml +++ b/tests/default/indices/index.yaml @@ -84,4 +84,12 @@ chapters: version: '>= 2.0' parameters: index: books,games - cluster_manager_timeout: 10s + cluster_manager_timeout: 10s + + - synopsis: Delete a nonexistent index `movies`. + path: /{index} + method: DELETE + parameters: + index: movies + response: + status: 404 diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 26dd02753..f85e6fdc6 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -62,7 +62,7 @@ export default class ChapterReader { response.status = e.response.status response.content_type = e.response.headers['content-type']?.split(';')[0] const payload = this.#deserialize_payload(e.response.data, response.content_type) - if (payload !== undefined) response.payload = payload.error + if (payload !== undefined) response.payload = payload.error ?? payload response.message = payload.error?.reason ?? e.response.statusText this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } diff --git a/tools/tests/tester/ChapterReader.test.ts b/tools/tests/tester/ChapterReader.test.ts index 7e1776bb2..38a0b82fc 100644 --- a/tools/tests/tester/ChapterReader.test.ts +++ b/tools/tests/tester/ChapterReader.test.ts @@ -219,6 +219,35 @@ describe('ChapterReader', () => { }] ]) }) + + it('sets payload to entire response when payload.error is missing', async () => { + const mock_payload = { '_data': '1', 'result': 'updated' }; + const mock_error = { + response: { + status: 404, + headers: { + 'content-type': 'application/json' + }, + data: JSON.stringify(mock_payload), + statusText: 'Not Found' + } + }; + + mocked_axios.request.mockRejectedValue(mock_error); + + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'POST' + }, new StoryOutputs()); + + expect(result).toStrictEqual({ + status: 404, + content_type: 'application/json', + payload: mock_payload, + message: 'Not Found' + }); + }); }) describe('deserialize_payload', () => { From 4d74a4908c3cd4549cae820852c0c19cafaf6be2 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Tue, 8 Oct 2024 00:53:32 +1300 Subject: [PATCH 38/73] Document individual expand wildcard values (#601) Signed-off-by: Thomas Farr --- spec/schemas/_common.yaml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 33b6ea830..471d2209a 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -406,13 +406,22 @@ components: items: $ref: '#/components/schemas/ExpandWildcard' ExpandWildcard: - type: string - enum: - - all - - closed - - hidden - - none - - open + oneOf: + - type: string + const: all + description: Match any index, including hidden ones. + - type: string + const: closed + description: Match closed, non-hidden indices. + - type: string + const: hidden + description: Match hidden indices. Must be combined with open, closed, or both. + - type: string + const: none + description: Wildcard expressions are not accepted. + - type: string + const: open + description: Match open, non-hidden indices. VersionString: type: string SearchType: From 10027be9609a6345cd82cd4840da86bdcba2f179 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Mon, 7 Oct 2024 15:08:31 -0500 Subject: [PATCH 39/73] [Docs] Fix field cap descriptions (#603) * Fix field cap descriptions Signed-off-by: Archer * Apply suggestions from code review Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower --- spec/schemas/_core.field_caps.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/schemas/_core.field_caps.yaml b/spec/schemas/_core.field_caps.yaml index f4cf82d15..70b93d6d5 100644 --- a/spec/schemas/_core.field_caps.yaml +++ b/spec/schemas/_core.field_caps.yaml @@ -10,7 +10,7 @@ components: type: object properties: aggregatable: - description: Whether this field can be aggregated on all indices. + description: Whether this field can be aggregated on all indexes. type: boolean indices: $ref: '_common.yaml#/components/schemas/Indices' @@ -21,7 +21,7 @@ components: non_searchable_indices: $ref: '_common.yaml#/components/schemas/Indices' searchable: - description: Whether this field is indexed for search on all indices. + description: Whether this field is indexed for search on all indexes. type: boolean type: type: string @@ -35,15 +35,15 @@ components: $ref: '_common.mapping.yaml#/components/schemas/TimeSeriesMetricType' non_dimension_indices: description: |- - If this list is present in response then some indices have the - field marked as a dimension and other indices, the ones in this list, do not. + If this list is present in the response, then indexes not contained in the list have the + field marked as a dimension. Any indexes contained in the list are not marked as a dimension. type: array items: $ref: '_common.yaml#/components/schemas/IndexName' metric_conflicts_indices: description: |- - The list of indices where this field is present if these indices - don't have the same `time_series_metric` value for this field. + The list of indexes in which this field is present if the indexes + don't have the same `time_series_metric` value for the field. type: array items: $ref: '_common.yaml#/components/schemas/IndexName' From 6b1d2fbfa918431a665ca712ff6a29ce6849578d Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:02:34 -0500 Subject: [PATCH 40/73] [Docs] Update msearch descriptions (#604) * Update msearch descriptions Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Include additional mterm vectors changes. Signed-off-by: Archer * Apply suggestions from code review Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower --- spec/schemas/_core.msearch.yaml | 38 ++++++++++++------------ spec/schemas/_core.msearch_template.yaml | 10 +++---- spec/schemas/_core.mtermvectors.yaml | 12 ++++---- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/spec/schemas/_core.msearch.yaml b/spec/schemas/_core.msearch.yaml index 06c256943..a63678983 100644 --- a/spec/schemas/_core.msearch.yaml +++ b/spec/schemas/_core.msearch.yaml @@ -58,8 +58,8 @@ components: $ref: '_common.yaml#/components/schemas/Fields' docvalue_fields: description: |- - Array of wildcard (*) patterns. The request returns doc values for field - names matching these patterns in the hits.fields property of the response. + An array of wildcard (*) patterns. The request returns document values for field + names matching these patterns in the `hits.fields` property of the response. type: array items: $ref: '_common.query_dsl.yaml#/components/schemas/FieldAndFormat' @@ -72,14 +72,14 @@ components: $ref: '_common.query_dsl.yaml#/components/schemas/KnnQuery' from: description: |- - Starting document offset. By default, you cannot page through more than 10,000 + The starting document offset. By default, you cannot page through more than 10,000 hits using the from and size parameters. To page through more hits, use the - search_after parameter. + `search_after` parameter. type: number highlight: $ref: '_core.search.yaml#/components/schemas/Highlight' indices_boost: - description: Boosts the _score of documents from specified indices. + description: Boosts the `_score` of documents from specified indexes. type: array items: type: object @@ -87,8 +87,8 @@ components: type: number min_score: description: |- - Minimum _score for matching documents. Documents with a lower _score are - not included in the search results. + The minimum `_score` for document matching. Documents with a lower `_score` than the minimum + are not included in the search results. type: number post_filter: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' @@ -119,22 +119,22 @@ components: $ref: '_core.search.yaml#/components/schemas/SourceConfig' fields: description: |- - Array of wildcard (*) patterns. The request returns values for field names - matching these patterns in the hits.fields property of the response. + An array of wildcard (*) patterns. The request returns values for field names + matching these patterns in the `hits.fields` property of the response. type: array items: $ref: '_common.query_dsl.yaml#/components/schemas/FieldAndFormat' terminate_after: description: |- - Maximum number of documents to collect for each shard. If a query reaches this - limit, OpenSearch terminates the query early. OpenSearch collects documents - before sorting. Defaults to 0, which does not terminate query execution early. + The maximum number of documents to collect for each shard. If a query reaches this + limit, OpenSearch stops the query early. OpenSearch collects documents + before sorting. Default is `0`, which does not terminate query execution early. type: number stats: description: |- - Stats groups to associate with the search. Each group maintains a statistics - aggregation for its associated searches. You can retrieve these stats using - the indices stats API. + The statistics groups to associate with the search. Each group maintains a statistics + aggregation for its associated searches. You can retrieve these statistics using + the Index Stats API. type: array items: type: string @@ -142,21 +142,21 @@ components: description: |- Specifies the period of time to wait for a response from each shard. If no response is received before the timeout expires, the request fails and returns an error. - Defaults to no timeout. + Default is no timeout. type: string track_scores: - description: If true, calculate and return document scores, even if the scores are not used for sorting. + description: When true, calculates and returns all document scores, even if the scores are not used for sorting. type: boolean track_total_hits: $ref: '_core.search.yaml#/components/schemas/TrackHits' version: - description: If true, returns document version as part of a hit. + description: When true, returns the document version as part of the hit. type: boolean runtime_mappings: $ref: '_common.mapping.yaml#/components/schemas/RuntimeFields' seq_no_primary_term: description: |- - If true, returns sequence number and primary term of the last modification + When true, returns the sequence number and primary term of the last modification of each hit. See Optimistic concurrency control. type: boolean pit: diff --git a/spec/schemas/_core.msearch_template.yaml b/spec/schemas/_core.msearch_template.yaml index 23a050708..3b6966d0a 100644 --- a/spec/schemas/_core.msearch_template.yaml +++ b/spec/schemas/_core.msearch_template.yaml @@ -14,24 +14,24 @@ components: type: object properties: explain: - description: If `true`, returns detailed information about score calculation as part of each hit. + description: When `true`, returns detailed information about score calculations as part of each hit. type: boolean id: $ref: '_common.yaml#/components/schemas/Id' params: description: |- - Key-value pairs used to replace Mustache variables in the template. + The key-value pairs used to replace any Mustache variables in the template. The key is the variable name. The value is the variable value. type: object additionalProperties: type: object profile: - description: If `true`, the query execution is profiled. + description: When `true`, provides a profile for the query execution. type: boolean source: description: |- - An inline search template. Supports the same parameters as the search API's - request body. Also supports Mustache variables. If no id is specified, this + An inline search template that supports the same parameters as the search API's + request body. Also supports Mustache variables. When no template ID is specified, this parameter is required. type: string diff --git a/spec/schemas/_core.mtermvectors.yaml b/spec/schemas/_core.mtermvectors.yaml index bed09977d..02b168921 100644 --- a/spec/schemas/_core.mtermvectors.yaml +++ b/spec/schemas/_core.mtermvectors.yaml @@ -14,28 +14,28 @@ components: _index: $ref: '_common.yaml#/components/schemas/IndexName' doc: - description: An artificial document (a document not present in the index) for which you want to retrieve term vectors. + description: An artificial document (a document not present in the index) for which you want to retrieve term vectors. type: object fields: $ref: '_common.yaml#/components/schemas/Fields' field_statistics: - description: If `true`, the response includes the document count, sum of document frequencies, and sum of total term frequencies. + description: When `true`, the response includes the document count, the sum of document frequencies, and the sum of term frequencies. type: boolean filter: $ref: '_core.termvectors.yaml#/components/schemas/Filter' offsets: - description: If `true`, the response includes term offsets. + description: When `true`, the response includes term offsets. type: boolean payloads: - description: If `true`, the response includes term payloads. + description: When `true`, the response includes term payloads. type: boolean positions: - description: If `true`, the response includes term positions. + description: When `true`, the response includes term positions. type: boolean routing: $ref: '_common.yaml#/components/schemas/Routing' term_statistics: - description: If true, the response includes term frequency and document frequency. + description: When `true`, the response includes the term frequency and the document frequency. type: boolean version: $ref: '_common.yaml#/components/schemas/VersionNumber' From 6845fc7824ed3a259c6f8b710ba52be45b7df92c Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:03:41 -0500 Subject: [PATCH 41/73] [Docs] Update common schema descriptions (#602) * Update common schema descriptions Signed-off-by: Archer * Fix errors Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Fix lint and validation errors. Signed-off-by: Archer * Fix typo Signed-off-by: Archer * Fix last error Signed-off-by: Archer * Apply suggestions from code review Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Fix remaining editorial items Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Add additional description. Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Fix scalar pipes Signed-off-by: Archer * Fix scalar pipe Signed-off-by: Archer * Test other scalar pipes. Signed-off-by: Archer * Revert scalar pipes back to previous. Signed-off-by: Archer * Fix duration patterns. Signed-off-by: Archer --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower Co-authored-by: Daniel (dB.) Doubrovkine --- spec/schemas/_common.yaml | 262 +++++++++++++++++++------------------- 1 file changed, 131 insertions(+), 131 deletions(-) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 471d2209a..df0f5158f 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -21,7 +21,7 @@ components: - acknowledged DocStatus: description: |- - Item level REST category class codes during indexing. + The item level REST category class codes during indexing. type: object properties: 1xx: @@ -45,11 +45,11 @@ components: additionalProperties: true GeoBounds: description: |- - A geo bounding box. It can be represented in various ways: - - as 4 top/bottom/left/right coordinates - - as 2 top_left / bottom_right points - - as 2 top_right / bottom_left points - - as a WKT bounding box. + A geo-bounding box. It can be represented in the following ways: + - As 4 top/bottom/left/right coordinates. + - As 2 top_left/bottom_right points. + - As 2 top_right/bottom_left points. + - As a Well Known Text (WKT) bounding box. oneOf: - title: coords $ref: '#/components/schemas/CoordsGeoBounds' @@ -91,11 +91,11 @@ components: - top_left GeoLocation: description: |- - A latitude/longitude as a 2-dimensional point. It can be represented in various ways: - - as a `{lat, long}` object - - as a geo hash value - - as a `[lon, lat]` array - - as a string in `", "` or WKT point formats. + A latitude/longitude as a two-dimensional point. It can be represented in the following ways: + - As a `{lat, long}` object. + - As a geohash value. + - As a `[lon, lat]` array. + - As a string in `", "` or WKT point format. oneOf: - title: latlon $ref: '#/components/schemas/LatLonGeoLocation' @@ -151,10 +151,10 @@ components: XyLocation: x-version-added: '2.4' description: |- - A two-dimensional Cartesian point specified by x and y coordinates. It can be represented in various ways: - - as a `{x, y}` object - - as a `[x, y]` array - - as a string in `"x, y"` or WKT point formats. + A two-dimensional Cartesian point specified by x and y coordinates. It can be represented in the following ways: + - As an `{x, y}` object. + - As an `[x, y]` array. + - As a string in `"x, y"` or WKT point format. oneOf: - title: cartesian $ref: '#/components/schemas/XyCartesianCoordinates' @@ -181,12 +181,12 @@ components: allOf: - $ref: '#/components/schemas/UnitMillis' UnitMillis: - description: Time unit for milliseconds. + description: The time unit for milliseconds. type: integer format: int64 DurationLarge: description: |- - A date histogram interval. Similar to `Duration` with additional units: `w` (week), `M` (month), `q` (quarter) and `y` (year). + A date histogram interval, similar to `Duration`, with support for additional units: `w` (week), `M` (month), `q` (quarter), and `y` (year). type: string FieldValue: description: A field value. @@ -200,8 +200,8 @@ components: description: |- The absence of any type. This is commonly used in APIs that don't return a body. - Although "void" is generally used for the unit type that has only one value, this is to be interpreted as - the bottom type that has no value at all. Most languages have a unit type, but few have a bottom type. + Although "void" is generally used for a unit type that has only one value, this is interpreted as + the bottom type, which has no value. Most languages have a unit type, but few have a bottom type. See https://en.m.wikipedia.org/wiki/Unit_type and https://en.m.wikipedia.org/wiki/Bottom_type. type: object @@ -214,7 +214,7 @@ components: IndexName: type: string Field: - description: Path to field or array of paths. Some API's support wildcards in the path to select multiple fields. + description: The path to a field or an array of paths. Some APIs support wildcards in the path, which allows you to select multiple fields. type: string SequenceNumber: type: integer @@ -233,7 +233,7 @@ components: description: Always `"LineString"` type: string coordinates: - description: Array of `[lon, lat]` coordinates + description: An array of `[lon, lat]` coordinates. type: array items: type: array @@ -341,13 +341,13 @@ components: type: object properties: type: - description: The type of error + description: The type of error. type: string reason: - description: A human-readable explanation of the error, in english + description: A human-readable explanation of the error, in English. type: string stack_trace: - description: The server stack trace. Present only if the `error_trace=true` parameter was sent with the request. + description: The server stack trace, present only if the `error_trace=true` parameter was sent with the request. type: string caused_by: $ref: '#/components/schemas/ErrorCause' @@ -363,7 +363,7 @@ components: - type additionalProperties: title: metadata - description: Additional details about the error. + description: Any additional information about the error. DurationValueUnitNanos: allOf: - $ref: '#/components/schemas/UnitNanos' @@ -381,8 +381,8 @@ components: type: string DateTime: description: |- - A date and time, either as a string whose format can depend on the context (defaulting to ISO 8601), or a - number of milliseconds since the Epoch. OpenSearch accepts both as input, but will generally output a string + A date and time, either as a string whose format depends on the context (defaulting to ISO_8601) or the + number of milliseconds since the epoch. OpenSearch accepts both as an input but will generally output a string. representation. oneOf: - type: string @@ -436,7 +436,7 @@ components: - missing - popular MinimumShouldMatch: - description: The minimum number of terms that should match as integer, percentage or range. + description: The minimum number of terms that should match as an integer, percentage, or range. oneOf: - type: number - type: string @@ -478,8 +478,8 @@ components: properties: params: description: |- - Specifies any named parameters that are passed into the script as variables. - Use parameters instead of hard-coded values to decrease compile time. + Specifies any named parameters that are passed into the script as variables. + Use parameters instead of hard-coded values to decrease compilation time. type: object additionalProperties: true StoredScriptId: @@ -649,8 +649,8 @@ components: type: string PipeSeparatedFlagsSimpleQueryStringFlag: description: |- - A set of flags that can be represented as a single enum value or a set of values that are encoded - as a pipe-separated string + A set of flags represented as a single enum value or a set of values that are encoded + as a pipe-separated string. Depending on the target language, code generators can use this hint to generate language specific flags enum constructs and the corresponding (de-)serialization code. @@ -658,7 +658,7 @@ components: - $ref: '_common.query_dsl.yaml#/components/schemas/SimpleQueryStringFlag' - type: string GeoHashPrecision: - description: A precision that can be expressed as a geohash length between 1 and 12, or a distance measure like "1km", "10m". + description: The level of geohash precision, which can be expressed as a geohash length between 1 and 12 or as a distance measure, such as "1km" or "10m". oneOf: - type: number - type: string @@ -672,25 +672,25 @@ components: vector: $ref: '#/components/schemas/QueryVector' k: - description: The final number of nearest neighbors to return as top hits. + description: The total number of nearest neighbors to return as top hits. type: number min_score: - description: The minimum similarity score for a neighbor to be considered a hit. + description: The minimum similarity score required in order for a neighbor to be considered a hit. type: number x-version-added: '2.14' max_distance: - description: The maximum physical distance in vector space for a neighbor to be considered a hit. + description: The maximum physical vector space distance required in order for a neighbor to be considered a hit. type: number x-version-added: '2.14' filter: - description: Filters for the kNN search query. + description: The filters for the k-NN search query. oneOf: - $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' - type: array items: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' boost: - description: Boost value to apply to kNN scores + description: The boost value applied to k-NN scores. type: number method_parameters: type: object @@ -802,8 +802,8 @@ components: type: string StringifiedEpochTimeUnitSeconds: description: |- - Some APIs will return values such as numbers also as a string (notably epoch timestamps). This behavior - is used to capture this behavior while keeping the semantics of the field type. + Certain APIs may return values, including numbers such as epoch timestamps, as strings. This setting captures + this behavior while keeping the semantics of the field type. Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. @@ -873,8 +873,8 @@ components: type: string Stringifiedinteger: description: |- - Some APIs will return values such as numbers also as a string (notably epoch timestamps). This behavior - is used to capture this behavior while keeping the semantics of the field type. + Certain APIs may return values, including numbers such as epoch timestamps, as strings. This setting captures + this behavior while keeping the semantics of the field type. Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. @@ -883,8 +883,8 @@ components: - type: string Stringifiedboolean: description: |- - Some APIs will return values such as numbers also as a string (notably epoch timestamps). This behavior - is used to capture this behavior while keeping the semantics of the field type. + Certain APIs may return values, including numbers such as epoch timestamps, as strings. This setting captures + this behavior while keeping the semantics of the field type. Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. @@ -895,8 +895,8 @@ components: type: string StringifiedEpochTimeUnitMillis: description: |- - Some APIs will return values such as numbers also as a string (notably epoch timestamps). This behavior - is used to capture this behavior while keeping the semantics of the field type. + Certain APIs may return values, including numbers such as epoch timestamps, as strings. This setting captures + this behavior while keeping the semantics of the field type. Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. @@ -970,7 +970,7 @@ components: type: object properties: size_in_bytes: - description: Total amount, in bytes, of memory used for completion across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for completion across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' size: $ref: '#/components/schemas/HumanReadableByteCount' @@ -994,14 +994,14 @@ components: properties: count: description: |- - Total number of non-deleted documents across all primary shards assigned to selected nodes. + The total number of non-deleted documents across all primary shards assigned to the selected nodes. This number is based on documents in Lucene segments and may include documents from nested fields. type: number deleted: description: |- - Total number of deleted documents across all primary shards assigned to selected nodes. - This number is based on documents in Lucene segments. - OpenSearch reclaims the disk space of deleted Lucene documents when a segment is merged. + The total number of deleted documents across all primary shards assigned to the selected nodes. + This number is based on the number of documents stored in Lucene segments. + OpenSearch reclaims the disk space previously occupied by the deleted Lucene documents when a segment is merged. type: number required: - count @@ -1034,28 +1034,28 @@ components: properties: cache_count: description: |- - Total number of entries added to the query cache across all shards assigned to selected nodes. - This number includes current and evicted entries. + The total number of entries added to the query cache across all shards assigned to the selected nodes. + This number includes all current and evicted entries. type: number cache_size: - description: Total number of entries currently in the query cache across all shards assigned to selected nodes. + description: The total number of entries currently stored in the query cache across all shards assigned to the selected nodes. type: number evictions: - description: Total number of query cache evictions across all shards assigned to selected nodes. + description: The total number of query cache evictions across all shards assigned to the selected nodes. type: number hit_count: - description: Total count of query cache hits across all shards assigned to selected nodes. + description: The total number of query cache hits across all shards assigned to the selected nodes. type: number memory_size: $ref: '#/components/schemas/HumanReadableByteCount' memory_size_in_bytes: - description: Total amount, in bytes, of memory used for the query cache across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for the query cache across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' miss_count: - description: Total count of query cache misses across all shards assigned to selected nodes. + description: The total number of query cache misses across all shards assigned to the selected nodes. type: number total_count: - description: Total count of hits and misses in the query cache across all shards assigned to selected nodes. + description: The total number of hits and misses stored in the query cache across all shards assigned to the selected nodes. type: number required: - cache_count @@ -1067,7 +1067,7 @@ components: - total_count RemoteStoreStats: type: object - description: Statistics about remote segment store operations. + description: Statistics related to remote segment store operations. properties: upload: $ref: '#/components/schemas/RemoteStoreUploadStats' @@ -1113,7 +1113,7 @@ components: - succeeded RemoteStoreTranslogUploadTotalUploadSizeStats: type: object - description: The total amount of data uploaded to the remote translog store. + description: The total amount of data uploaded to the remote translog store. properties: failed: description: The number of bytes that failed to upload to the remote translog store. @@ -1152,7 +1152,7 @@ components: refresh_size_lag: $ref: '#/components/schemas/RemoteStoreUploadRefreshSizeLagStats' total_time_spent: - description: The total amount of time, spent on uploads to the remote segment store. + description: The total amount of time spent on uploads to the remote segment store. $ref: '#/components/schemas/Duration' total_time_spent_in_millis: description: The total amount of time, in milliseconds, spent on uploads to the remote segment store. @@ -1170,10 +1170,10 @@ components: description: The amount of data, in bytes, uploaded or downloaded to/from the remote segment store. properties: failed: - description: The number of bytes that failed to upload/upload to/from the remote segment store. + description: The number of bytes that failed to upload to/from the remote segment store. $ref: '#/components/schemas/HumanReadableByteCount' failed_bytes: - description: The number of bytes that failed to upload/upload to/from the remote segment store. + description: The number of bytes that failed to upload to/from the remote segment store. $ref: '#/components/schemas/ByteCount' started: description: The number of bytes to upload/download to/from the remote segment store after the upload/download has started. @@ -1193,10 +1193,10 @@ components: - succeeded_bytes RemoteStoreUploadPressureStats: type: object - description: Statistics related to segment store upload backpressure. + description: Statistics related to segment store upload backpressure. properties: total_rejections: - description: The total number of requests rejected due to segment store upload backpressure. + description: The total number of requests rejected due to segment store upload backpressure. type: number required: - total_rejections @@ -1205,32 +1205,32 @@ components: description: The amount of lag during upload between the remote segment store and the local store. properties: max: - description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. + description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. $ref: '#/components/schemas/HumanReadableByteCount' max_bytes: - description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. + description: The maximum amount of lag, in bytes, during the upload refresh between the remote segment store and the local store. $ref: '#/components/schemas/ByteCount' total: - description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. + description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. $ref: '#/components/schemas/HumanReadableByteCount' total_bytes: - description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. + description: The total number of bytes that lagged during the upload refresh between the remote segment store and the local store. $ref: '#/components/schemas/ByteCount' required: - max_bytes - total_bytes RemoteStoreDownloadStats: type: object - description: Statistics related to downloads to the remote segment store. + description: Statistics related to downloads to the remote segment store. properties: total_download_size: - description: The total amount of data download from the remote segment store. + description: The total amount of data downloaded from the remote segment store. $ref: '#/components/schemas/RemoteStoreUploadDownloadStats' total_time_spent: - description: The total duration, spent on downloads from the remote segment store. + description: The total amount of time spent on downloads from the remote segment store. $ref: '#/components/schemas/Duration' total_time_spent_in_millis: - description: The total duration, in milliseconds, spent on downloads from the remote segment store. + description: The total duration, in milliseconds, spent on downloads from the remote segment store. $ref: '#/components/schemas/DurationValueUnitMillis' required: - total_download_size @@ -1252,12 +1252,12 @@ components: type: object properties: count: - description: Total number of segments across all shards assigned to selected nodes. + description: The total number of segments across all shards assigned to the selected nodes. type: number doc_values_memory: $ref: '#/components/schemas/HumanReadableByteCount' doc_values_memory_in_bytes: - description: Total amount, in bytes, of memory used for doc values across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for document values across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' file_sizes: description: |- @@ -1269,59 +1269,59 @@ components: fixed_bit_set: $ref: '#/components/schemas/HumanReadableByteCount' fixed_bit_set_memory_in_bytes: - description: Total amount of memory, in bytes, used by fixed bit sets across all shards assigned to selected nodes. + description: The total amount of memory, in bytes, used by fixed bit sets across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' index_writer_memory: $ref: '#/components/schemas/HumanReadableByteCount' index_writer_max_memory_in_bytes: $ref: '#/components/schemas/ByteCount' index_writer_memory_in_bytes: - description: Total amount, in bytes, of memory used by all index writers across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used by all index writers across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' max_unsafe_auto_id_timestamp: - description: Unix timestamp, in milliseconds, of the most recently retried indexing request. + description: The Unix timestamp, in milliseconds, of the most recently retried indexing request. type: number memory: - description: Total amount, of memory used for segments across all shards assigned to selected nodes. + description: The total amount of memory used for segments across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' memory_in_bytes: - description: Total amount, in bytes, of memory used for segments across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for segments across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' norms_memory: - description: Total amount of memory used for normalization factors across all shards assigned to selected nodes. + description: The total amount of memory used for normalization factors across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' norms_memory_in_bytes: - description: Total amount, in bytes, of memory used for normalization factors across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for normalization factors across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' points_memory: - description: Total amount of memory used for points across all shards assigned to selected nodes. + description: The total amount of memory used for points across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' points_memory_in_bytes: - description: Total amount, in bytes, of memory used for points across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for points across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' stored_fields_memory: - description: Total amount of memory used for stored fields across all shards assigned to selected nodes. + description: The total amount of memory used for stored fields across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' stored_fields_memory_in_bytes: - description: Total amount, in bytes, of memory used for stored fields across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for stored fields across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' terms_memory: - description: Total amount of memory used for terms across all shards assigned to selected nodes. + description: The total amount of memory used for terms across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' terms_memory_in_bytes: - description: Total amount, in bytes, of memory used for terms across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for terms across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' term_vectors_memory: - description: Total amount of memory used for term vectors across all shards assigned to selected nodes. + description: The total amount of memory used for term vectors across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' term_vectors_memory_in_bytes: - description: Total amount, in bytes, of memory used for term vectors across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used for term vectors across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' version_map_memory: - description: Total amount of memory used by all version maps across all shards assigned to selected nodes. + description: The total amount of memory used by all version maps across all shards assigned to the selected nodes. $ref: '#/components/schemas/HumanReadableByteCount' version_map_memory_in_bytes: - description: Total amount, in bytes, of memory used by all version maps across all shards assigned to selected nodes. + description: The total amount, in bytes, of memory used by all version maps across all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' remote_store: $ref: '#/components/schemas/RemoteStoreStats' @@ -1347,7 +1347,7 @@ components: size: $ref: '#/components/schemas/HumanReadableByteCount' size_in_bytes: - description: Total size, in bytes, of all shards assigned to selected nodes. + description: The total size, in bytes, of all shards assigned to the selected nodes. $ref: '#/components/schemas/ByteCount' reserved: $ref: '#/components/schemas/HumanReadableByteCount' @@ -1399,13 +1399,13 @@ components: items: $ref: '#/components/schemas/ErrorCause' total: - description: Total number of nodes selected by the request. + description: The total number of nodes selected by the request. type: integer successful: - description: Number of nodes that responded successfully to the request. + description: The number of nodes that responded successfully to the request. type: integer failed: - description: Number of nodes that rejected the request or failed to respond. If this value is not 0, a reason for the rejection or failure is included in the response. + description: The number of nodes that rejected the request or failed to respond. If this value is not 0, then a reason for the rejection or failure is included in the response. type: integer required: - failed @@ -1455,7 +1455,7 @@ components: - abort - proceed Slices: - description: Slices configuration used to parallelize a process. + description: The slice configuration used to parallelize a process. oneOf: - type: number - $ref: '#/components/schemas/SlicesCalculation' @@ -1767,42 +1767,42 @@ components: type: object properties: open_contexts: - description: The number of open search contexts. + description: The number of open search contexts. type: number query_current: - description: The number of shard query operations that are currently running. + description: The number of currently running shard query operations. type: number query_time: - description: The total amount of time for all shard query operations. + description: The total amount of time taken to complete all shard query operations. $ref: '#/components/schemas/Duration' query_time_in_millis: - description: The total amount of time for all shard query operations, in milliseconds. + description: The total amount of time taken to complete all shard query operations, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' query_total: description: The total number of shard query operations. type: number concurrent_query_total: - description: The total number of query operations that use concurrent segment search. + description: The total number of query operations using concurrent segment search. type: number concurrent_query_time: $ref: '#/components/schemas/Duration' concurrent_query_time_in_millis: - description: The total amount of time taken by all query operations that use concurrent segment search, in milliseconds. + description: The total amount of time taken by all query operations using concurrent segment search, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' concurrent_query_current: - description: The number of currently running query operations that use concurrent segment search. + description: The number of currently running query operations using concurrent segment search. type: number concurrent_avg_slice_count: description: The average slice count of all search requests. This is computed as the total slice count divided by the total number of concurrent search requests. type: number fetch_current: - description: The number of shard fetch operations that are currently running. + description: The number of currently running shard fetch operations. type: number fetch_time: - description: The total amount of time for all shard fetch operations. + description: The total amount of time taken to complete all shard fetch operations. $ref: '#/components/schemas/Duration' fetch_time_in_millis: - description: The total amount of time for all shard fetch operations, in milliseconds. + description: The total amount of time taken to complete all shard fetch operations, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' fetch_total: description: The total number of shard fetch operations. @@ -1811,42 +1811,42 @@ components: description: The number of shard scroll operations that are currently running. type: number scroll_time: - description: The total amount of time for all shard scroll operations. + description: The total amount of time taken to complete all shard scroll operations. $ref: '#/components/schemas/Duration' scroll_time_in_millis: - description: The total amount of time for all shard scroll operations, in milliseconds. + description: The total amount of time taken to complete all shard scroll operations, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' scroll_total: description: The total number of shard scroll operations. type: number point_in_time_total: - description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. + description: The total number of shard Point in Time (PIT) contexts created (completed and active) since the node last restarted. type: number point_in_time_time: $ref: '#/components/schemas/Duration' point_in_time_time_in_millis: - description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. + description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' point_in_time_current: - description: The number of shard PIT contexts currently open. + description: The number of currently open shard PIT contexts. type: number suggest_current: - description: The number of shard suggest operations that are currently running. + description: The number of currently running shard suggest operations. type: number suggest_time: - description: The total amount of time for all shard suggest operations. + description: The total amount of time take to complete all shard suggest operations. $ref: '#/components/schemas/Duration' suggest_time_in_millis: - description: The total amount of time for all shard suggest operations, in milliseconds. + description: The total amount of time taken to complete all shard suggest operations, in milliseconds. $ref: '#/components/schemas/DurationValueUnitMillis' suggest_total: - description: The total number of shard suggest operations. + description: The total number of shard suggest operations. type: number search_idle_reactivate_count_total: type: number request: type: object - description: Statistics about coordinator search operations for the node. + description: Statistics related to coordinator search operations for the node. additionalProperties: $ref: '#/components/schemas/RequestStats' groups: @@ -1926,8 +1926,8 @@ components: - within StringifiedVersionNumber: description: |- - Some APIs will return values such as numbers also as a string (notably epoch timestamps). This behavior - is used to capture this behavior while keeping the semantics of the field type. + Certain APIs may return values, including numbers such as epoch timestamps, as strings. This setting captures + this behavior while keeping the semantics of the field type. Depending on the target language, code generators can keep the union or remove it and leniently parse strings to the target type. @@ -1993,7 +1993,7 @@ components: - master deprecated: true x-version-deprecated: '2.0' - x-deprecation-message: To promote inclusive language, use 'cluster_manager' instead. + x-deprecation-message: Use 'cluster_manager' instead. - type: string enum: - cluster_manager @@ -2042,10 +2042,10 @@ components: - type: object properties: rank_constant: - description: How much influence documents in individual result sets per query have over the final ranked result set + description: To what degree documents found in individual result sets per query influence the final ranked result set. type: number window_size: - description: Size of the individual result sets per query + description: The size of the individual result sets per query. type: number RankBase: type: object @@ -2152,31 +2152,31 @@ components: type: integer format: int32 total: - description: The number of documents that were successfully processed. + description: The number of documents that were successfully processed. type: integer format: int64 updated: - description: The number of documents that were successfully updated, for example, a document with same ID already existed prior to reindex updating it. + description: The number of documents that were successfully updated after the reindex operation. type: integer format: int64 created: - description: The number of documents that were successfully created. + description: The number of documents that were successfully created. type: integer format: int64 deleted: - description: The number of documents that were successfully deleted. + description: The number of documents that were successfully deleted. type: integer format: int64 batches: - description: The number of scroll responses pulled back by the reindex. + description: The number of scroll responses pulled back by the reindex operation. type: integer format: int32 version_conflicts: - description: The number of version conflicts that reindex hits. + description: The number of version conflicts encountered by the reindex operation. type: integer format: int64 noops: - description: The number of documents that were ignored. + description: The number of documents that were ignored. type: integer format: int64 retries: @@ -2186,7 +2186,7 @@ components: throttled: $ref: '#/components/schemas/Duration' requests_per_second: - description: The number of requests per second effectively executed during the reindex. + description: The number of requests per second effectively executed during the reindex operation. type: number format: float canceled: From 6b637597197a46c2393d640cc8b8694989f842e7 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Thu, 10 Oct 2024 01:25:02 +1300 Subject: [PATCH 42/73] Allow shortcut specifying InlineScript as just a string (#605) * Allow shortcut specifying InlineScript as just a string Signed-off-by: Thomas Farr * Add test Signed-off-by: Thomas Farr * Allow arbitrary script language Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/schemas/_common.yaml | 39 ++++++---- tests/default/_core/search/query/script.yaml | 78 ++++++++++++++++++++ 3 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 tests/default/_core/search/query/script.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index b323df068..f92d956a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586)) - Added `persian_stem` filter ([#592](https://github.com/opensearch-project/opensearch-api-specification/pull/592)) - Added `404` response for `DELETE /{index}`, `GET /{index}/_doc/{id}`, `DELETE /{index}/_doc/{id}` ([#589](https://github.com/opensearch-project/opensearch-api-specification/pull/589)) +- Added ability to pass `InlineScript` as a simple string ([#605](https://github.com/opensearch-project/opensearch-api-specification/pull/605)) ### Changed diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index df0f5158f..9b3f07813 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -451,22 +451,31 @@ components: - title: stored $ref: '#/components/schemas/StoredScriptId' InlineScript: - allOf: - - $ref: '#/components/schemas/ScriptBase' - - type: object - properties: - lang: - $ref: '#/components/schemas/ScriptLanguage' - options: - type: object - additionalProperties: - type: string - source: - description: The script source. - type: string - required: - - source + oneOf: + - title: source + type: string + - allOf: + - $ref: '#/components/schemas/ScriptBase' + - type: object + properties: + lang: + $ref: '#/components/schemas/ScriptLanguage' + options: + type: object + additionalProperties: + type: string + source: + description: The script source. + type: string + required: + - source ScriptLanguage: + anyOf: + - title: builtin + $ref: '#/components/schemas/BuiltinScriptLanguage' + - title: custom + type: string + BuiltinScriptLanguage: type: string enum: - expression diff --git a/tests/default/_core/search/query/script.yaml b/tests/default/_core/search/query/script.yaml new file mode 100644 index 000000000..525cddc8c --- /dev/null +++ b/tests/default/_core/search/query/script.yaml @@ -0,0 +1,78 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test ScriptQuery functionality. + +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + title: + type: keyword + year: + type: integer + + - path: /movies/_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - index: + _id: 1 + - title: The Lion King + year: 1994 + - index: + _id: 2 + - title: Beauty and the Beast + year: 1991 + - index: + _id: 3 + - title: Aladdin + year: 1992 + - index: + _id: 4 + - title: The Little Mermaid + year: 1989 + status: [200] + +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Search using ScriptQuery with filtering to movies with odd years. + path: /{index}/_search + parameters: + index: movies + method: POST + request: + payload: + query: + bool: + filter: + script: + script: "doc['year'].value % 2 == 1" + sort: + - year: + order: desc + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 2 + hits: + - _id: '2' + _source: + title: Beauty and the Beast + year: 1991 + - _id: '4' + _source: + title: The Little Mermaid + year: 1989 From 67f522164d948c660d980511cf1f8da8964e6f8a Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 9 Oct 2024 15:52:11 -0400 Subject: [PATCH 43/73] Fix: specify _source includes/excludes. (#607) Signed-off-by: dblock --- spec/schemas/_core.search.yaml | 2 +- tests/default/_core/mget.yaml | 47 ++++++++++++++++++++++++++++++++ tests/default/indices/mget.yaml | 48 +++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 tests/default/_core/mget.yaml create mode 100644 tests/default/indices/mget.yaml diff --git a/spec/schemas/_core.search.yaml b/spec/schemas/_core.search.yaml index d56bb06c8..549c6f58e 100644 --- a/spec/schemas/_core.search.yaml +++ b/spec/schemas/_core.search.yaml @@ -791,7 +791,7 @@ components: - title: includes type: array items: - $ref: '_common.yaml#/components/schemas/Field' + $ref: '_common.yaml#/components/schemas/Field' - title: filter $ref: '#/components/schemas/SourceFilter' SourceFilter: diff --git a/tests/default/_core/mget.yaml b/tests/default/_core/mget.yaml new file mode 100644 index 000000000..42f1afcf7 --- /dev/null +++ b/tests/default/_core/mget.yaml @@ -0,0 +1,47 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test document multi-get. +epilogues: + - path: /books + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book1}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {create: {_index: movies, _id: movie1}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies, _id: movie2}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Retrieve documents from multiple indexes. + path: /_mget + method: GET + request: + payload: + docs: + - _index: movies + _id: movie1 + _source: true + - _index: books + _id: book1 + _source: + includes: + - title + response: + status: 200 + payload: + docs: + - _index: movies + _id: movie1 + - _index: books + _id: book1 diff --git a/tests/default/indices/mget.yaml b/tests/default/indices/mget.yaml new file mode 100644 index 000000000..9b81f3dd8 --- /dev/null +++ b/tests/default/indices/mget.yaml @@ -0,0 +1,48 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test document multi-get. +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies, _id: movie1}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies, _id: movie2}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Retrieve documents from multiple indexes. + path: /{index}/_mget + method: GET + parameters: + index: movies + request: + payload: + docs: + - _id: movie1 + _source: + excludes: + - title + - _id: movie1 + _source: false + - _id: movie2 + _source: + includes: + - title + response: + status: 200 + payload: + docs: + - _index: movies + _id: movie1 + - _index: movies + _id: movie1 + - _index: movies + _id: movie2 From 49b7c6895871209a489cccd7d3eb4fabba8e241c Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 9 Oct 2024 15:54:23 -0400 Subject: [PATCH 44/73] Added tests for the notifications plugin. (#594) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 + .lycheeignore | 1 + CHANGELOG.md | 2 + spec/namespaces/notifications.yaml | 16 +++ spec/schemas/notifications._common.yaml | 25 ++--- tests/default/notifications/channels.yaml | 10 ++ tests/default/notifications/configs.yaml | 97 +++++++++++++++++++ tests/default/notifications/features.yaml | 10 ++ .../plugins/notifications/docker-compose.yml | 19 ++++ .../notifications/feature/test.yaml | 37 +++++++ tests/plugins/notifications/server.py | 31 ++++++ tools/src/tester/TestRunner.ts | 2 +- tools/tests/tester/integ/TestRunner.test.ts | 6 ++ 13 files changed, 240 insertions(+), 18 deletions(-) create mode 100644 tests/default/notifications/channels.yaml create mode 100644 tests/default/notifications/configs.yaml create mode 100644 tests/default/notifications/features.yaml create mode 100644 tests/plugins/notifications/docker-compose.yml create mode 100644 tests/plugins/notifications/notifications/feature/test.yaml create mode 100644 tests/plugins/notifications/server.py diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 06a098265..a4c354c89 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -38,6 +38,8 @@ jobs: tests: snapshot - version: 2.17.0 tests: plugins/streaming + - version: 2.17.0 + tests: plugins/notifications - version: 2.18.0 hub: opensearchstaging ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' diff --git a/.lycheeignore b/.lycheeignore index c859dd4da..caa2546d0 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1 +1,2 @@ https://localhost:* +http://webhook:8080 diff --git a/CHANGELOG.md b/CHANGELOG.md index f92d956a6..c6fe8e15b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `persian_stem` filter ([#592](https://github.com/opensearch-project/opensearch-api-specification/pull/592)) - Added `404` response for `DELETE /{index}`, `GET /{index}/_doc/{id}`, `DELETE /{index}/_doc/{id}` ([#589](https://github.com/opensearch-project/opensearch-api-specification/pull/589)) - Added ability to pass `InlineScript` as a simple string ([#605](https://github.com/opensearch-project/opensearch-api-specification/pull/605)) +- Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) ### Changed @@ -174,6 +175,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) - Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598))([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Fixed mapping and analysis types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) +- Fixed `RestStatus` responses in `DELETE /_plugins/_notifications/configs/{config_id}` ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) ### Security diff --git a/spec/namespaces/notifications.yaml b/spec/namespaces/notifications.yaml index 00faf3941..811a2f21b 100644 --- a/spec/namespaces/notifications.yaml +++ b/spec/namespaces/notifications.yaml @@ -26,6 +26,8 @@ paths: parameters: - $ref: '#/components/parameters/notifications.get_configs::query.chime.url' - $ref: '#/components/parameters/notifications.get_configs::query.chime.url.keyword' + - $ref: '#/components/parameters/notifications.get_configs::query.config_id' + - $ref: '#/components/parameters/notifications.get_configs::query.config_id_list' - $ref: '#/components/parameters/notifications.get_configs::query.config_type' - $ref: '#/components/parameters/notifications.get_configs::query.created_time_ms' - $ref: '#/components/parameters/notifications.get_configs::query.description' @@ -318,6 +320,20 @@ components: in: query schema: type: string + notifications.get_configs::query.config_id: + name: config_id + in: query + description: Notification configuration ID. + schema: + type: string + notifications.get_configs::query.config_id_list: + name: config_id_list + in: query + description: Notification configuration IDs. + schema: + type: array + items: + type: string notifications.get_configs::query.config_type: name: config_type in: query diff --git a/spec/schemas/notifications._common.yaml b/spec/schemas/notifications._common.yaml index 55d1a0040..667eb0847 100644 --- a/spec/schemas/notifications._common.yaml +++ b/spec/schemas/notifications._common.yaml @@ -34,23 +34,14 @@ components: RestStatus: type: string enum: - - accepted - - continue - - created - - found - - moved_permanently - - multi_status - - multiple_choices - - no_content - - non_authoritative_information - - not_modified - - ok - - partial_content - - reset_content - - see_other - - switching_protocols - - temporary_redirect - - use_proxy + - ACCEPTED + - CREATED + - MULTI_STATUS + - NON_AUTHORITATIVE_INFORMATION + - NO_CONTENT + - OK + - PARTIAL_CONTENT + - RESET_CONTENT NotificationConfigType: type: string description: Type of notification configuration. diff --git a/tests/default/notifications/channels.yaml b/tests/default/notifications/channels.yaml new file mode 100644 index 000000000..d45fea828 --- /dev/null +++ b/tests/default/notifications/channels.yaml @@ -0,0 +1,10 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test listing channels. +version: '>= 2.0' +chapters: + - synopsis: Retrieve a list of all notification channels. + path: /_plugins/_notifications/channels + method: GET + response: + status: 200 diff --git a/tests/default/notifications/configs.yaml b/tests/default/notifications/configs.yaml new file mode 100644 index 000000000..3a1523f1e --- /dev/null +++ b/tests/default/notifications/configs.yaml @@ -0,0 +1,97 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test listing notification configs. +version: '>= 2.0' +chapters: + - synopsis: Create a channel configuration. + path: /_plugins/_notifications/configs + method: POST + request: + payload: + config_id: webhook-configuration + config: + name: Notifications Channel + description: Default notifications channel. + config_type: webhook + is_enabled: true + webhook: + url: http://webhook:8080/ + response: + status: 200 + - synopsis: Update a channel configuration. + path: /_plugins/_notifications/configs/{config_id} + method: PUT + parameters: + config_id: webhook-configuration + request: + payload: + config: + name: Notifications Channel + config_type: webhook + is_enabled: true + webhook: + url: http://webhook:8080/ + response: + status: 200 + - synopsis: Retrieve a list of all notification configurations. + path: /_plugins/_notifications/configs + method: GET + retry: + count: 2 + response: + status: 200 + payload: + config_list: + - config_id: webhook-configuration + - synopsis: Retrieve a list of all notification configurations filtered by config ID. + path: /_plugins/_notifications/configs + method: GET + parameters: + config_type: webhook + response: + status: 200 + payload: + config_list: + - config_id: webhook-configuration + - synopsis: Retrieve a notification configuration by config ID (path). + path: /_plugins/_notifications/configs/{config_id} + method: GET + parameters: + config_id: webhook-configuration + response: + status: 200 + payload: + config_list: + - config_id: webhook-configuration + - synopsis: Retrieve a notification configuration by config ID (query). + path: /_plugins/_notifications/configs + method: GET + parameters: + config_id: webhook-configuration + response: + status: 200 + payload: + config_list: + - config_id: webhook-configuration + - synopsis: Retrieve a notification configuration by config ID list (query). + path: /_plugins/_notifications/configs + method: GET + parameters: + config_id_list: + - webhook-configuration + response: + status: 200 + payload: + config_list: + - config_id: webhook-configuration + - synopsis: Delete a channel configuration. + path: /_plugins/_notifications/configs/{config_id} + method: DELETE + parameters: + config_id: webhook-configuration + response: + status: 200 +epilogues: + - path: /_plugins/_notifications/configs/webhook-configuration + method: DELETE + status: [200,404] diff --git a/tests/default/notifications/features.yaml b/tests/default/notifications/features.yaml new file mode 100644 index 000000000..6c11b90b1 --- /dev/null +++ b/tests/default/notifications/features.yaml @@ -0,0 +1,10 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test listing supported channel configurations. +version: '>= 2.0' +chapters: + - synopsis: Retrieve a list of all supported notification configuration types. + path: /_plugins/_notifications/features + method: GET + response: + status: 200 diff --git a/tests/plugins/notifications/docker-compose.yml b/tests/plugins/notifications/docker-compose.yml new file mode 100644 index 000000000..937846f3c --- /dev/null +++ b/tests/plugins/notifications/docker-compose.yml @@ -0,0 +1,19 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node + webhook: + image: python:latest + volumes: + - ./server.py:/server.py + ports: + - '8080:8080' + entrypoint: python server.py diff --git a/tests/plugins/notifications/notifications/feature/test.yaml b/tests/plugins/notifications/notifications/feature/test.yaml new file mode 100644 index 000000000..e889155e1 --- /dev/null +++ b/tests/plugins/notifications/notifications/feature/test.yaml @@ -0,0 +1,37 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test sending a notification. +version: '>= 2.0' +prologues: + - path: /_plugins/_notifications/configs + method: POST + request: + payload: + config_id: custom-webhook-configuration + config: + name: Notifications Channel + description: Default notifications channel. + config_type: webhook + is_enabled: true + webhook: + url: http://webhook:8080/ + status: [200] +chapters: + - synopsis: Test sending a notification. + path: /_plugins/_notifications/feature/test/{config_id} + method: GET + parameters: + config_id: custom-webhook-configuration + response: + status: 200 + payload: + status_list: + - config_id: custom-webhook-configuration + config_type: webhook + delivery_status: + status_code: '200' + status_text: '{"ok":"true"}' +epilogues: + - path: /_plugins/_notifications/configs/custom-webhook-configuration + method: DELETE + status: [200,404] diff --git a/tests/plugins/notifications/server.py b/tests/plugins/notifications/server.py new file mode 100644 index 000000000..20de61d94 --- /dev/null +++ b/tests/plugins/notifications/server.py @@ -0,0 +1,31 @@ +# +# Copyright OpenSearch Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# + +from http.server import BaseHTTPRequestHandler, HTTPServer + + +class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header("Content-type", "application_json") + self.end_headers() + self.wfile.write(b'{"ok":"true"}') + + def do_POST(self): + self.send_response(200) + self.send_header("Content-type", "application_json") + self.end_headers() + self.wfile.write(b'{"ok":"true"}') + + +if __name__ == "__main__": + server_address = ("", 8080) + httpd = HTTPServer(server_address, SimpleHTTPRequestHandler) + print("Server started on http://localhost:8080") + httpd.serve_forever() diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 7b36af3b9..6317ef392 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -70,7 +70,7 @@ export default class TestRunner { #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (file.startsWith('.') || file == 'docker-compose.yml' || file == 'Dockerfile') { + if (file.startsWith('.') || file == 'docker-compose.yml' || file == 'Dockerfile' || file.endsWith('.py')) { return [] } else if (fs.statSync(path).isFile()) { const story: Story = read_yaml(path) diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 75253cd7c..a747f1b64 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -53,4 +53,10 @@ describe('story_files', () => { story_file => story_file.display_path )).not.toContain('nodes/plugins/docker-compose.yml') }) + + test('does not contain a python script', () => { + expect(test_runner.story_files('tests/plugins/notifications').map( + story_file => story_file.display_path + )).not.toContain('nodes/plugins/server.py') + }) }) From 5dc65a531c5de10d423670a259c4b1905bd3407e Mon Sep 17 00:00:00 2001 From: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:58:14 +0900 Subject: [PATCH 45/73] fix spec snapshot endpoint (#608) * fix spec snapshot endpoint Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> * fix lint and add changelog Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> --------- Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> --- CHANGELOG.md | 1 + spec/namespaces/snapshot.yaml | 13 +---- spec/schemas/snapshot._common.yaml | 39 ++----------- spec/schemas/snapshot.get.yaml | 21 ------- tests/snapshot/snapshot/snapshot.yaml | 79 +++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 68 deletions(-) delete mode 100644 spec/schemas/snapshot.get.yaml create mode 100644 tests/snapshot/snapshot/snapshot.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index c6fe8e15b..13a7e05fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -176,6 +176,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598))([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Fixed mapping and analysis types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Fixed `RestStatus` responses in `DELETE /_plugins/_notifications/configs/{config_id}` ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) +- Fixed `GET /_snapshot_/{repository}/{snapshot}` ([#608](https://github.com/opensearch-project/opensearch-api-specification/pull/608)) ### Security diff --git a/spec/namespaces/snapshot.yaml b/spec/namespaces/snapshot.yaml index 5a8bbfb92..58ad5f82e 100644 --- a/spec/namespaces/snapshot.yaml +++ b/spec/namespaces/snapshot.yaml @@ -408,23 +408,12 @@ components: schema: type: object properties: - responses: - type: array - items: - $ref: '../schemas/snapshot.get.yaml#/components/schemas/SnapshotResponseItem' snapshots: type: array items: $ref: '../schemas/snapshot._common.yaml#/components/schemas/SnapshotInfo' - total: - description: The total number of snapshots that match the request when ignoring size limit or after query parameter. - type: number - remaining: - description: The number of remaining snapshots that were not returned due to size limits and that can be fetched by additional requests using the next field value. - type: number required: - - remaining - - total + - snapshots snapshot.get_repository@200: content: application/json: diff --git a/spec/schemas/snapshot._common.yaml b/spec/schemas/snapshot._common.yaml index 83c8a8d9c..b6caade7f 100644 --- a/spec/schemas/snapshot._common.yaml +++ b/spec/schemas/snapshot._common.yaml @@ -31,16 +31,14 @@ components: type: array items: $ref: '_common.yaml#/components/schemas/IndexName' - index_details: - type: object - additionalProperties: - $ref: '#/components/schemas/IndexDetails' metadata: $ref: '_common.yaml#/components/schemas/Metadata' + pinned_timestamp: + $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' reason: type: string - repository: - $ref: '_common.yaml#/components/schemas/Name' + remote_store_index_shallow_copy: + type: boolean snapshot: $ref: '_common.yaml#/components/schemas/Name' shards: @@ -57,10 +55,6 @@ components: $ref: '_common.yaml#/components/schemas/VersionString' version_id: $ref: '_common.yaml#/components/schemas/VersionNumber' - feature_states: - type: array - items: - $ref: '#/components/schemas/InfoFeatureState' required: - data_streams - snapshot @@ -83,31 +77,6 @@ components: - reason - shard_id - status - IndexDetails: - type: object - properties: - shard_count: - type: number - size: - $ref: '_common.yaml#/components/schemas/HumanReadableByteCount' - size_in_bytes: - $ref: '_common.yaml#/components/schemas/ByteCount' - max_segments_per_shard: - type: number - required: - - max_segments_per_shard - - shard_count - - size_in_bytes - InfoFeatureState: - type: object - properties: - feature_name: - type: string - indices: - $ref: '_common.yaml#/components/schemas/Indices' - required: - - feature_name - - indices Repository: type: object properties: diff --git a/spec/schemas/snapshot.get.yaml b/spec/schemas/snapshot.get.yaml deleted file mode 100644 index 8834816d5..000000000 --- a/spec/schemas/snapshot.get.yaml +++ /dev/null @@ -1,21 +0,0 @@ -openapi: 3.1.0 -info: - title: Schemas of snapshot.get category - description: Schemas of snapshot.get category - version: 1.0.0 -paths: {} -components: - schemas: - SnapshotResponseItem: - type: object - properties: - repository: - $ref: '_common.yaml#/components/schemas/Name' - snapshots: - type: array - items: - $ref: 'snapshot._common.yaml#/components/schemas/SnapshotInfo' - error: - $ref: '_common.yaml#/components/schemas/ErrorCause' - required: - - repository diff --git a/tests/snapshot/snapshot/snapshot.yaml b/tests/snapshot/snapshot/snapshot.yaml new file mode 100644 index 000000000..ca77a39bb --- /dev/null +++ b/tests/snapshot/snapshot/snapshot.yaml @@ -0,0 +1,79 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test _snapshot/{repository}/snapshot endpoints. +epilogues: + - path: /_snapshot/{repository}/{snapshot} + method: DELETE + status: [200, 404] + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + - path: /_snapshot/{repository} + method: DELETE + status: [200, 404] + parameters: + repository: my-fs-repository +prologues: + - path: /_snapshot/{repository} + method: PUT + parameters: + repository: my-fs-repository + request: + payload: + type: fs + settings: + location: /tmp/opensearch/repo +chapters: + - synopsis: Get information about a repository. + path: /_snapshot/{repository} + method: GET + parameters: + repository: my-fs-repository + response: + status: 200 + payload: + my-fs-repository: + type: fs + settings: + location: /tmp/opensearch/repo + - synopsis: Create a snapshot. + path: /_snapshot/{repository}/{snapshot} + method: PUT + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + wait_for_completion: true + request: + payload: + indices: '*' + ignore_unavailable: true + include_global_state: false + partial: true + response: + status: 200 + payload: + snapshot: + snapshot: my-test-snapshot + - synopsis: Get information about a snapshot. + path: /_snapshot/{repository}/{snapshot} + method: GET + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + response: + status: 200 + payload: + snapshots: + - snapshot: my-test-snapshot + - synopsis: Get information about a snapshot with human-readable parameter. + path: /_snapshot/{repository}/{snapshot} + method: GET + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + human: true + response: + status: 200 + payload: + snapshots: + - snapshot: my-test-snapshot From 051c21dfc2d1bcd456b453859fe0f8f4aad55274 Mon Sep 17 00:00:00 2001 From: Shuyi Z <159193473+amberzsy@users.noreply.github.com> Date: Mon, 14 Oct 2024 06:00:54 -0700 Subject: [PATCH 46/73] update with accurate numeric type for common, core (#612) * update with accurate numeric type for common, core Signed-off-by: amberzsy * add changelog Signed-off-by: amberzsy * update explian value field with 'General Number' Signed-off-by: amberzsy --------- Signed-off-by: amberzsy --- CHANGELOG.md | 2 +- spec/schemas/_common.query_dsl.yaml | 3 +- spec/schemas/_common.yaml | 65 +++-- spec/schemas/_core.bulk.yaml | 12 +- spec/schemas/_core.explain.yaml | 20 +- spec/schemas/_core.search.yaml | 244 ++++++++++++------ spec/schemas/asynchronous_search._common.yaml | 3 +- 7 files changed, 244 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 13a7e05fb..98eddac78 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -173,7 +173,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582)) - Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588)) - Fixed `/{index}/_search` with aggregations ([#576](https://github.com/opensearch-project/opensearch-api-specification/pull/576)) -- Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598))([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) +- Fixed inaccurate numeric type ([#597](https://github.com/opensearch-project/opensearch-api-specification/pull/597))([#598](https://github.com/opensearch-project/opensearch-api-specification/pull/598))([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600))([#612](https://github.com/opensearch-project/opensearch-api-specification/pull/612)) - Fixed mapping and analysis types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Fixed `RestStatus` responses in `DELETE /_plugins/_notifications/configs/{config_id}` ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Fixed `GET /_snapshot_/{repository}/{snapshot}` ([#608](https://github.com/opensearch-project/opensearch-api-specification/pull/608)) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 872ecc039..070d84ce8 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -1824,7 +1824,8 @@ components: description: Can only be used as a clause in a span_near query. type: object additionalProperties: - type: number + type: integer + format: int32 minProperties: 1 maxProperties: 1 SpanMultiTermQuery: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 9b3f07813..6f2b287d3 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -25,15 +25,20 @@ components: type: object properties: 1xx: - type: number + type: integer + format: int32 2xx: - type: number + type: integer + format: int32 3xx: - type: number + type: integer + format: int32 4xx: - type: number + type: integer + format: int32 5xx: - type: number + type: integer + format: int32 Duration: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and @@ -162,6 +167,7 @@ components: type: array items: type: number + format: double - title: text type: string XyCartesianCoordinates: @@ -171,9 +177,11 @@ components: x: description: The x coordinate. type: number + format: double y: description: The y coordinate. type: number + format: double required: - x - y @@ -239,6 +247,7 @@ components: type: array items: type: number + format: double required: - coordinates - type @@ -246,17 +255,23 @@ components: type: object properties: skipped: - type: number + type: integer + format: int32 successful: - type: number + type: integer + format: int32 total: - type: number + type: integer + format: int32 running: - type: number + type: integer + format: int32 partial: - type: number + type: integer + format: int32 failed: - type: number + type: integer + format: int32 details: type: object additionalProperties: @@ -438,7 +453,8 @@ components: MinimumShouldMatch: description: The minimum number of terms that should match as an integer, percentage, or range. oneOf: - - type: number + - type: integer + format: int32 - type: string Distance: type: string @@ -512,7 +528,8 @@ components: Fuzziness: oneOf: - type: string - - type: number + - type: integer + format: int32 GeoDistanceType: type: string enum: @@ -631,6 +648,7 @@ components: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' max_children: type: integer + format: int32 nested: $ref: '#/components/schemas/NestedSortValue' path: @@ -669,7 +687,8 @@ components: GeoHashPrecision: description: The level of geohash precision, which can be expressed as a geohash length between 1 and 12 or as a distance measure, such as "1km" or "10m". oneOf: - - type: number + - type: integer + format: int32 - type: string GeoTilePrecision: type: number @@ -682,14 +701,17 @@ components: $ref: '#/components/schemas/QueryVector' k: description: The total number of nearest neighbors to return as top hits. - type: number + type: integer + format: int32 min_score: description: The minimum similarity score required in order for a neighbor to be considered a hit. type: number + format: float x-version-added: '2.14' max_distance: description: The maximum physical vector space distance required in order for a neighbor to be considered a hit. type: number + format: float x-version-added: '2.14' filter: description: The filters for the k-NN search query. @@ -701,22 +723,26 @@ components: boost: description: The boost value applied to k-NN scores. type: number + format: float method_parameters: type: object x-version-added: '2.16' additionalProperties: - type: number + type: integer + format: int32 rescore: type: object x-version-added: '2.17' additionalProperties: type: number + format: float required: - vector QueryVector: type: array items: type: number + format: float SlicedScroll: type: object properties: @@ -725,7 +751,8 @@ components: id: $ref: '#/components/schemas/Id' max: - type: number + type: integer + format: int32 required: - id - max @@ -758,7 +785,8 @@ components: _seq_no: $ref: '#/components/schemas/SequenceNumber' _primary_term: - type: number + type: integer + format: int32 _routing: $ref: '#/components/schemas/Routing' _source: @@ -802,6 +830,7 @@ components: format: int64 PercentageNumber: type: number + format: double PercentageString: type: string pattern: '\d+(\.\d+)?' diff --git a/spec/schemas/_core.bulk.yaml b/spec/schemas/_core.bulk.yaml index 5db0782c8..92e0a9200 100644 --- a/spec/schemas/_core.bulk.yaml +++ b/spec/schemas/_core.bulk.yaml @@ -53,7 +53,8 @@ components: routing: $ref: '_common.yaml#/components/schemas/Routing' if_primary_term: - type: number + type: integer + format: int64 if_seq_no: $ref: '_common.yaml#/components/schemas/SequenceNumber' version: @@ -71,7 +72,8 @@ components: description: When `true`, the request's actions must target an index alias. type: boolean retry_on_conflict: - type: number + type: integer + format: int32 DeleteOperation: $ref: '#/components/schemas/OperationBase' UpdateAction: @@ -117,12 +119,14 @@ components: type: string status: description: HTTP status code returned for the operation. - type: number + type: integer + format: int32 error: $ref: '_common.yaml#/components/schemas/ErrorCause' _primary_term: description: The primary term assigned to the document for the operation. - type: number + type: integer + format: int64 result: description: |- Result of the operation. diff --git a/spec/schemas/_core.explain.yaml b/spec/schemas/_core.explain.yaml index c0821a79b..f1e354551 100644 --- a/spec/schemas/_core.explain.yaml +++ b/spec/schemas/_core.explain.yaml @@ -16,7 +16,15 @@ components: items: $ref: '#/components/schemas/ExplanationDetail' value: - type: number + oneOf: + - type: integer + format: int32 + - type: integer + format: int64 + - type: number + format: float + - type: number + format: double required: - description - details @@ -31,7 +39,15 @@ components: items: $ref: '#/components/schemas/ExplanationDetail' value: - type: number + oneOf: + - type: integer + format: int32 + - type: integer + format: int64 + - type: number + format: float + - type: number + format: double required: - description - value diff --git a/spec/schemas/_core.search.yaml b/spec/schemas/_core.search.yaml index 549c6f58e..a9d1f9077 100644 --- a/spec/schemas/_core.search.yaml +++ b/spec/schemas/_core.search.yaml @@ -13,7 +13,8 @@ components: description: Total hit count information, present only if `track_total_hits` wasn't `false` in the search request. oneOf: - $ref: '#/components/schemas/TotalHits' - - type: number + - type: integer + format: int64 hits: type: array items: @@ -28,7 +29,8 @@ components: relation: $ref: '#/components/schemas/TotalHitsRelation' value: - type: number + type: integer + format: int64 required: - relation - value @@ -91,7 +93,8 @@ components: _seq_no: $ref: '_common.yaml#/components/schemas/SequenceNumber' _primary_term: - type: number + type: integer + format: int64 _version: $ref: '_common.yaml#/components/schemas/VersionNumber' sort: @@ -112,7 +115,8 @@ components: field: $ref: '_common.yaml#/components/schemas/Field' offset: - type: number + type: integer + format: int32 _nested: $ref: '#/components/schemas/NestedIdentity' required: @@ -172,29 +176,41 @@ components: type: object properties: build_aggregation: - type: number + type: integer + format: int64 build_aggregation_count: - type: number + type: integer + format: int64 build_leaf_collector: - type: number + type: integer + format: int64 build_leaf_collector_count: - type: number + type: integer + format: int64 collect: - type: number + type: integer + format: int64 collect_count: - type: number + type: integer + format: int64 initialize: - type: number + type: integer + format: int64 initialize_count: - type: number + type: integer + format: int64 post_collection: - type: number + type: integer + format: int64 post_collection_count: - type: number + type: integer + format: int64 reduce: - type: number + type: integer + format: int64 reduce_count: - type: number + type: integer + format: int64 required: - build_aggregation - build_aggregation_count @@ -210,15 +226,19 @@ components: type: object properties: segments_with_multi_valued_ords: - type: number + type: integer + format: int32 collection_strategy: type: string segments_with_single_valued_ords: - type: number + type: integer + format: int32 total_buckets: - type: number + type: integer + format: int32 built_buckets: - type: number + type: integer + format: int32 result_strategy: type: string has_filter: @@ -228,58 +248,76 @@ components: delegate_debug: $ref: '#/components/schemas/AggregationProfileDebug' chars_fetched: - type: number + type: integer + format: int32 extract_count: - type: number + type: integer + format: int32 extract_ns: - type: number + type: integer + format: int32 values_fetched: - type: number + type: integer + format: int32 collect_analyzed_ns: - type: number + type: integer + format: int32 collect_analyzed_count: - type: number + type: integer + format: int32 surviving_buckets: - type: number + type: integer + format: int32 ordinals_collectors_used: - type: number + type: integer + format: int32 ordinals_collectors_overhead_too_high: - type: number + type: integer + format: int32 string_hashing_collectors_used: - type: number + type: integer + format: int32 numeric_collectors_used: - type: number + type: integer + format: int32 empty_collectors_used: - type: number + type: integer + format: int32 deferred_aggregators: type: array items: type: string segments_with_doc_count_field: - type: number + type: integer + format: int32 segments_with_deleted_docs: - type: number + type: integer + format: int32 filters: type: array items: $ref: '#/components/schemas/AggregationProfileDelegateDebugFilter' segments_counted: - type: number + type: integer + format: int32 segments_collected: - type: number + type: integer + format: int32 map_reducer: type: string AggregationProfileDelegateDebugFilter: type: object properties: results_from_metadata: - type: number + type: integer + format: int32 query: type: string specialized_for: type: string segments_counted_in_constant_time: - type: number + type: integer + format: int32 SearchProfile: type: object properties: @@ -292,7 +330,8 @@ components: items: $ref: '#/components/schemas/QueryProfile' rewrite_time: - type: number + type: integer + format: int64 required: - collector - query @@ -338,41 +377,59 @@ components: type: object properties: advance: - type: number + type: integer + format: int64 advance_count: - type: number + type: integer + format: int64 build_scorer: - type: number + type: integer + format: int64 build_scorer_count: - type: number + type: integer + format: int64 create_weight: - type: number + type: integer + format: int64 create_weight_count: - type: number + type: integer + format: int64 match: - type: number + type: integer + format: int64 match_count: - type: number + type: integer + format: int64 shallow_advance: - type: number + type: integer + format: int64 shallow_advance_count: - type: number + type: integer + format: int64 next_doc: - type: number + type: integer + format: int64 next_doc_count: - type: number + type: integer + format: int64 score: - type: number + type: integer + format: int64 score_count: - type: number + type: integer + format: int64 compute_max_score: - type: number + type: integer + format: int64 compute_max_score_count: - type: number + type: integer + format: int64 set_min_competitive_score: - type: number + type: integer + format: int64 set_min_competitive_score_count: - type: number + type: integer + format: int64 required: - advance - advance_count @@ -422,17 +479,23 @@ components: load_source_count: type: number load_stored_fields: - type: number + type: integer + format: int32 load_stored_fields_count: - type: number + type: integer + format: int32 next_reader: - type: number + type: integer + format: int32 next_reader_count: - type: number + type: integer + format: int32 process_count: - type: number + type: integer + format: int32 process: - type: number + type: integer + format: int32 FetchProfileDebug: type: object properties: @@ -441,7 +504,8 @@ components: items: type: string fast_path: - type: number + type: integer + format: int32 Suggest: oneOf: - $ref: '#/components/schemas/CompletionSuggest' @@ -483,12 +547,14 @@ components: $ref: '_common.yaml#/components/schemas/Routing' _score: type: number + format: float _source: type: object text: type: string score: type: number + format: float required: - text Context: @@ -500,9 +566,11 @@ components: type: object properties: length: - type: number + type: integer + format: int32 offset: - type: number + type: integer + format: int32 text: type: string required: @@ -529,6 +597,7 @@ components: type: string score: type: number + format: float highlighted: type: string collate_match: @@ -556,8 +625,10 @@ components: type: string score: type: number + format: float freq: type: number + format: double highlighted: type: string collate_match: @@ -574,7 +645,8 @@ components: Defaults to 10,000 hits. oneOf: - type: boolean - - type: number + - type: integer + format: int32 SourceConfigParam: description: |- Defines how to fetch a source. Fetching can be disabled entirely, or the source can be filtered. @@ -590,9 +662,11 @@ components: size: description: The maximum number of hits to return per `inner_hits`. type: integer + format: int32 from: description: Inner hit starting document offset. type: integer + format: int32 collapse: $ref: '#/components/schemas/FieldCollapse' docvalue_fields: @@ -664,7 +738,8 @@ components: - type: object properties: fragment_offset: - type: number + type: integer + format: int32 matched_fields: $ref: '_common.yaml#/components/schemas/Fields' analyzer: @@ -679,7 +754,8 @@ components: type: string boundary_max_scan: description: How far to scan for boundary characters. - type: number + type: integer + format: int32 boundary_scanner: $ref: '#/components/schemas/BoundaryScanner' boundary_scanner_locale: @@ -694,22 +770,26 @@ components: $ref: '#/components/schemas/HighlighterFragmenter' fragment_size: description: The size of the highlighted fragment in characters. - type: number + type: integer + format: int32 highlight_filter: type: boolean highlight_query: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' max_fragment_length: - type: number + type: integer + format: int32 max_analyzed_offset: description: |- If set to a non-negative value, highlighting stops at this defined maximum limit. The rest of the text is not processed, thus not highlighted and no error is returned The `max_analyzed_offset` query setting does not override the `index.highlight.max_analyzed_offset` setting, which prevails when it's set to lower value than the query setting. - type: number + type: integer + format: int32 no_match_size: description: The amount of text you want to return from the beginning of the field if there are no matching fragments to highlight. - type: number + type: integer + format: int32 number_of_fragments: description: |- The maximum number of fragments to return. @@ -717,7 +797,8 @@ components: Instead, the entire field contents are highlighted and returned. This can be handy when you need to highlight short texts such as a title or address, but fragmentation is not required. If `number_of_fragments` is `0`, `fragment_size` is ignored. - type: number + type: integer + format: int32 options: type: object additionalProperties: @@ -730,7 +811,8 @@ components: Prevents the `fvh` highlighter from analyzing too many phrases and consuming too much memory. When using `matched_fields`, `phrase_limit` phrases per matched field are considered. Raising the limit increases query time and consumes more memory. Only supported by the `fvh` highlighter. - type: number + type: integer + format: int32 post_tags: description: |- Use in conjunction with `pre_tags` to define the HTML tags to use for the highlighted text. @@ -807,7 +889,8 @@ components: query: $ref: '#/components/schemas/RescoreQuery' window_size: - type: number + type: integer + format: int32 required: - query RescoreQuery: @@ -818,9 +901,11 @@ components: query_weight: description: Relative importance of the original query versus the rescore query. type: number + format: float rescore_query_weight: description: Relative importance of the rescore query versus the original query. type: number + format: float score_mode: $ref: '#/components/schemas/ScoreMode' required: @@ -852,7 +937,8 @@ components: type: object properties: took: - type: number + type: integer + format: int64 timed_out: type: boolean _shards: @@ -874,8 +960,10 @@ components: type: object max_score: type: number + format: float num_reduce_phases: - type: number + type: integer + format: int32 profile: $ref: '#/components/schemas/Profile' pit_id: diff --git a/spec/schemas/asynchronous_search._common.yaml b/spec/schemas/asynchronous_search._common.yaml index 17765d9f0..691bf9ba9 100644 --- a/spec/schemas/asynchronous_search._common.yaml +++ b/spec/schemas/asynchronous_search._common.yaml @@ -134,7 +134,8 @@ components: When possible, let OpenSearch perform early termination automatically. Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. If set to `0` (default), the query does not terminate early. - type: number + type: integer + format: int32 timeout: description: |- Specifies the period of time to wait for a response from each shard. From 99bc70159d4ea07d459ac9ecd64811de9076bcf8 Mon Sep 17 00:00:00 2001 From: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> Date: Wed, 16 Oct 2024 23:02:22 +0900 Subject: [PATCH 47/73] fix snapshot restore and recovery endpoint (#611) * fix snapshot restore and recovery endpoint Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> * add changelog Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> * Fixed index naming, description and changelog Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> --------- Signed-off-by: Tatsuya Kawakami <43780506+hogesako@users.noreply.github.com> --- CHANGELOG.md | 1 + spec/namespaces/snapshot.yaml | 7 +- spec/schemas/indices.recovery.yaml | 8 +++ tests/snapshot/snapshot/restore.yaml | 101 +++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 tests/snapshot/snapshot/restore.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 98eddac78..4ec8c1b80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -177,6 +177,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed mapping and analysis types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Fixed `RestStatus` responses in `DELETE /_plugins/_notifications/configs/{config_id}` ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Fixed `GET /_snapshot_/{repository}/{snapshot}` ([#608](https://github.com/opensearch-project/opensearch-api-specification/pull/608)) +- Fixed `POST /_snapshot/{repository}/{snapshot}/_restore` when `wait_for_completion` is `false` and `GET /{index}/_recovery` ([#611](https://github.com/opensearch-project/opensearch-api-specification/pull/611)) ### Security diff --git a/spec/namespaces/snapshot.yaml b/spec/namespaces/snapshot.yaml index 58ad5f82e..d81a5b173 100644 --- a/spec/namespaces/snapshot.yaml +++ b/spec/namespaces/snapshot.yaml @@ -383,7 +383,7 @@ components: type: object properties: accepted: - description: Equals `true` if the snapshot was accepted. Present when the request had `wait_for_completion` set to `false` + description: Returns `true` if the snapshot was accepted. Present when the request had `wait_for_completion` set to `false`. type: boolean snapshot: $ref: '../schemas/snapshot._common.yaml#/components/schemas/SnapshotInfo' @@ -427,10 +427,11 @@ components: schema: type: object properties: + accepted: + description: Returns `true` if the restore was accepted. Present when the request had `wait_for_completion` set to `false`. + type: boolean snapshot: $ref: '../schemas/snapshot.restore.yaml#/components/schemas/SnapshotRestore' - required: - - snapshot snapshot.status@200: content: application/json: diff --git a/spec/schemas/indices.recovery.yaml b/spec/schemas/indices.recovery.yaml index d2b781830..17019bfad 100644 --- a/spec/schemas/indices.recovery.yaml +++ b/spec/schemas/indices.recovery.yaml @@ -175,6 +175,14 @@ components: $ref: '_common.yaml#/components/schemas/Uuid' index: $ref: '_common.yaml#/components/schemas/IndexName' + isSearchableSnapshot: + type: boolean + remoteStoreIndexShallowCopy: + type: boolean + sourceRemoteStoreRepository: + type: ['null', string] + sourceRemoteTranslogRepository: + type: ['null', string] RecoveryStartStatus: type: object properties: diff --git a/tests/snapshot/snapshot/restore.yaml b/tests/snapshot/snapshot/restore.yaml new file mode 100644 index 000000000..970a39f00 --- /dev/null +++ b/tests/snapshot/snapshot/restore.yaml @@ -0,0 +1,101 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test _snapshot/{repository}/{snapshot}/_restore endpoints. +epilogues: + - path: /_snapshot/{repository}/{snapshot} + method: DELETE + status: [200, 404] + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + - path: /_snapshot/{repository} + method: DELETE + status: [200, 404] + parameters: + repository: my-fs-repository + - path: /movies + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +prologues: + - path: /_snapshot/{repository} + method: PUT + parameters: + repository: my-fs-repository + request: + payload: + type: fs + settings: + location: /tmp/opensearch/repo + - path: /movies + method: PUT + - path: /books + method: PUT + - path: /_snapshot/{repository}/{snapshot} + method: PUT + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + wait_for_completion: true + request: + payload: + indices: + - books + - movies + ignore_unavailable: true + include_global_state: false + partial: true + - path: /movies + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: Restore snapshot with wait_for_completion true. + path: /_snapshot/{repository}/{snapshot}/_restore + method: POST + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + wait_for_completion: true + request: + payload: + indices: movies + response: + status: 200 + payload: + snapshot: + snapshot: my-test-snapshot + - synopsis: Restore snapshot with wait_for_completion false. + path: /_snapshot/{repository}/{snapshot}/_restore + method: POST + parameters: + repository: my-fs-repository + snapshot: my-test-snapshot + wait_for_completion: false + request: + payload: + indices: books + response: + status: 200 + payload: + accepted: true + - synopsis: Wait finish async restore. + path: /{index}/_recovery + warnings: + multiple-paths-detected: false + method: GET + parameters: + index: books + response: + status: 200 + payload: + books: + shards: + - stage: DONE + type: SNAPSHOT + retry: + count: 3 \ No newline at end of file From 4c48c26fc3674b251525aaad24ce87510c34bb33 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:36:49 -0500 Subject: [PATCH 48/73] [Docs] Edit msearch parameters descriptions (#618) * Edit msearch parameters descriptions Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower --- spec/schemas/_core.msearch.yaml | 20 ++++++++++---------- spec/schemas/_core.msearch_template.yaml | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/spec/schemas/_core.msearch.yaml b/spec/schemas/_core.msearch.yaml index a63678983..19e8c6c9a 100644 --- a/spec/schemas/_core.msearch.yaml +++ b/spec/schemas/_core.msearch.yaml @@ -47,10 +47,10 @@ components: query: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' explain: - description: If true, returns detailed information about score computation as part of a hit. + description: When `true`, returns detailed information about score computation as part of a hit. type: boolean ext: - description: Configuration of search extensions defined by OpenSearch plugins. + description: The configuration of search extensions defined by OpenSearch plugins. type: object additionalProperties: type: object @@ -73,13 +73,13 @@ components: from: description: |- The starting document offset. By default, you cannot page through more than 10,000 - hits using the from and size parameters. To page through more hits, use the + hits using the `from` and `size` parameters. To page through more than 10,000 hits, use the `search_after` parameter. type: number highlight: $ref: '_core.search.yaml#/components/schemas/Highlight' indices_boost: - description: Boosts the `_score` of documents from specified indexes. + description: Boosts the `_score` of documents from the specified indexes. type: array items: type: object @@ -101,7 +101,7 @@ components: items: $ref: '_core.search.yaml#/components/schemas/Rescore' script_fields: - description: Retrieve a script evaluation (based on different fields) for each hit. + description: Retrieves a script evaluation (based on different fields) for each hit. type: object additionalProperties: $ref: '_common.yaml#/components/schemas/ScriptField' @@ -110,8 +110,8 @@ components: size: description: |- The number of hits to return. By default, you cannot page through more - than 10,000 hits using the from and size parameters. To page through more - hits, use the search_after parameter. + than 10,000 hits using the `from` and `size` parameters. To page through more + hits, use the `search_after` parameter. type: number sort: $ref: '_common.yaml#/components/schemas/Sort' @@ -145,18 +145,18 @@ components: Default is no timeout. type: string track_scores: - description: When true, calculates and returns all document scores, even if the scores are not used for sorting. + description: When `true`, calculates and returns all document scores, even if the scores are not used for sorting. type: boolean track_total_hits: $ref: '_core.search.yaml#/components/schemas/TrackHits' version: - description: When true, returns the document version as part of the hit. + description: When `true`, returns the document version as part of the hit. type: boolean runtime_mappings: $ref: '_common.mapping.yaml#/components/schemas/RuntimeFields' seq_no_primary_term: description: |- - When true, returns the sequence number and primary term of the last modification + When `true`, returns the sequence number and primary term of the last modification. of each hit. See Optimistic concurrency control. type: boolean pit: diff --git a/spec/schemas/_core.msearch_template.yaml b/spec/schemas/_core.msearch_template.yaml index 3b6966d0a..762874379 100644 --- a/spec/schemas/_core.msearch_template.yaml +++ b/spec/schemas/_core.msearch_template.yaml @@ -14,13 +14,13 @@ components: type: object properties: explain: - description: When `true`, returns detailed information about score calculations as part of each hit. + description: When `true`, returns detailed information about score calculation as part of each hit. type: boolean id: $ref: '_common.yaml#/components/schemas/Id' params: description: |- - The key-value pairs used to replace any Mustache variables in the template. + The key-value pairs used to replace Mustache variables in the template. The key is the variable name. The value is the variable value. type: object @@ -31,7 +31,7 @@ components: type: boolean source: description: |- - An inline search template that supports the same parameters as the search API's - request body. Also supports Mustache variables. When no template ID is specified, this + An inline search template that supports the same parameters as the Search API's + request body and Mustache variables. If no `id` is specified, this parameter is required. type: string From d2733fec89e43ae22b7d83895aa63b806e6ca3ac Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:17:12 -0500 Subject: [PATCH 49/73] [Docs] Descriptions for rank eval and mterm parameters (#622) Signed-off-by: Archer --- spec/schemas/_core.mtermvectors.yaml | 10 +++++----- spec/schemas/_core.rank_eval.yaml | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/spec/schemas/_core.mtermvectors.yaml b/spec/schemas/_core.mtermvectors.yaml index 02b168921..ced45db7e 100644 --- a/spec/schemas/_core.mtermvectors.yaml +++ b/spec/schemas/_core.mtermvectors.yaml @@ -14,23 +14,23 @@ components: _index: $ref: '_common.yaml#/components/schemas/IndexName' doc: - description: An artificial document (a document not present in the index) for which you want to retrieve term vectors. + description: An artificial document for which you want to retrieve term vectors. type: object fields: $ref: '_common.yaml#/components/schemas/Fields' field_statistics: - description: When `true`, the response includes the document count, the sum of document frequencies, and the sum of term frequencies. + description: When `true`, the response includes the document count, the sum of the document frequencies, and the sum of the term frequencies. type: boolean filter: $ref: '_core.termvectors.yaml#/components/schemas/Filter' offsets: - description: When `true`, the response includes term offsets. + description: When `true`, the response includes the term offsets. type: boolean payloads: - description: When `true`, the response includes term payloads. + description: When `true`, the response includes the term payloads. type: boolean positions: - description: When `true`, the response includes term positions. + description: When `true`, the response includes the term positions. type: boolean routing: $ref: '_common.yaml#/components/schemas/Routing' diff --git a/spec/schemas/_core.rank_eval.yaml b/spec/schemas/_core.rank_eval.yaml index 4527f0c05..d5e1d6eb0 100644 --- a/spec/schemas/_core.rank_eval.yaml +++ b/spec/schemas/_core.rank_eval.yaml @@ -14,7 +14,7 @@ components: request: $ref: '#/components/schemas/RankEvalQuery' ratings: - description: List of document ratings + description: A list of document ratings. type: array items: $ref: '#/components/schemas/DocumentRating' @@ -45,7 +45,7 @@ components: _index: $ref: '_common.yaml#/components/schemas/IndexName' rating: - description: The document's relevance with regard to this search request. + description: The document's relevance with regard to the specified search request. type: number required: - _id @@ -70,7 +70,7 @@ components: - type: object properties: ignore_unlabeled: - description: Controls how unlabeled documents in the search results are counted. If set to true, unlabeled documents are ignored and neither count as relevant or irrelevant. Set to false (the default), they are treated as irrelevant. + description: Controls how unlabeled documents in the search results are counted. When `true`, unlabeled documents are ignored and do not count as relevant or irrelevant. When `false`, unlabeled documents are treated as irrelevant. type: boolean RankEvalMetricRatingThreshold: allOf: @@ -78,13 +78,13 @@ components: - type: object properties: relevant_rating_threshold: - description: Sets the rating threshold above which documents are considered to be "relevant". + description: Sets the rating threshold above which documents are considered to be relevant. type: number RankEvalMetricBase: type: object properties: k: - description: Sets the maximum number of documents retrieved per query. This value will act in place of the usual size parameter in the query. + description: Sets the maximum number of documents retrieved per query. This value replaces the usual `size` parameter in the query. type: number RankEvalMetricRecall: allOf: @@ -100,7 +100,7 @@ components: - type: object properties: normalize: - description: If set to true, this metric will calculate the Normalized DCG (https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG). + description: When `true`, this metric calculates the Normalized DCG (https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG). type: boolean RankEvalMetricExpectedReciprocalRank: allOf: @@ -116,20 +116,20 @@ components: type: object properties: metric_score: - description: The metric_score in the details section shows the contribution of this query to the global quality metric score + description: The `metric_score`, found in the `metric_details` section, shows the contribution of this query to the global quality metric score. type: number unrated_docs: - description: The unrated_docs section contains an _index and _id entry for each document in the search result for this query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents + description: The `unrated_docs` section contains an `_index` and `_id` entry for each document that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents. type: array items: $ref: '#/components/schemas/UnratedDocument' hits: - description: The hits section shows a grouping of the search results with their supplied ratings + description: The `hits` section shows a grouping of the search results with their supplied ratings type: array items: $ref: '#/components/schemas/RankEvalHitItem' metric_details: - description: The metric_details give additional information about the calculated quality metric (e.g. how many of the retrieved documents were relevant). The content varies for each metric but allows for better interpretation of the results + description: The `metric_details` section gives additional information about the calculated quality metric, in other words, how many of the retrieved documents were relevant. The content varies for each metric but allows for better interpretation of the results. type: object additionalProperties: type: object From 7855ecafc6eefea60c5f0c0aa0cc8ca82008b894 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Sat, 19 Oct 2024 01:24:43 +1300 Subject: [PATCH 50/73] Differentiate between routing values in bodies versus query string (#626) Signed-off-by: Thomas Farr --- spec/namespaces/_core.yaml | 42 +++++++++++++++++++------------------- spec/namespaces/knn.yaml | 2 +- spec/schemas/_common.yaml | 2 ++ 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 79d1343e7..d96136a14 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -3369,7 +3369,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form bulk::query.timeout: in: query @@ -3456,7 +3456,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form bulk_stream::query.timeout: in: query @@ -3628,7 +3628,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form count::query.terminate_after: in: query @@ -3684,7 +3684,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form create::query.timeout: in: query @@ -3762,7 +3762,7 @@ components: description: Comma-separated list of specific routing values. style: form schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' explode: true delete::path.id: in: path @@ -3809,7 +3809,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form delete::query.timeout: in: query @@ -4024,7 +4024,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form delete_by_query::query.scroll: in: query @@ -4262,7 +4262,7 @@ components: name: routing description: Target the specified primary shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form exists::query.stored_fields: in: query @@ -4358,7 +4358,7 @@ components: name: routing description: Target the specified primary shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form exists_source::query.version: in: query @@ -4476,7 +4476,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form explain::query.stored_fields: in: query @@ -4596,7 +4596,7 @@ components: name: routing description: Target the specified primary shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form get::query.stored_fields: in: query @@ -4711,7 +4711,7 @@ components: name: routing description: Target the specified primary shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form get_source::query.version: in: query @@ -4803,7 +4803,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form index::query.timeout: in: query @@ -4899,7 +4899,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form mget::query.stored_fields: in: query @@ -5105,7 +5105,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form mtermvectors::query.term_statistics: in: query @@ -5609,7 +5609,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form search::query.cancel_after_time_interval: in: query @@ -5852,7 +5852,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form search_template::path.index: in: path @@ -5944,7 +5944,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form search_template::query.scroll: in: query @@ -6049,7 +6049,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form termvectors::query.term_statistics: in: query @@ -6167,7 +6167,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form update::query.timeout: in: query @@ -6379,7 +6379,7 @@ components: name: routing description: Custom value used to route operations to a specific shard. schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' style: form update_by_query::query.scroll: in: query diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index a33bd02df..c38bba5b7 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -514,7 +514,7 @@ components: description: Comma-separated list of specific routing values. style: form schema: - $ref: '../schemas/_common.yaml#/components/schemas/Routing' + $ref: '../schemas/_common.yaml#/components/schemas/RoutingInQueryString' explode: true knn.search_models::query.scroll: name: scroll diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 6f2b287d3..8b22e9880 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -389,6 +389,8 @@ components: ScrollId: type: string Routing: + type: string + RoutingInQueryString: oneOf: - type: string - type: array From e95e7d37d701674c90be2d0db920588e9bbfe1ec Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Tue, 22 Oct 2024 01:32:28 +1300 Subject: [PATCH 51/73] Add titles to WaitForActiveShards variants (#631) Signed-off-by: Thomas Farr --- spec/schemas/_common.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 8b22e9880..6450e9c6d 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -768,8 +768,10 @@ components: - wait_for WaitForActiveShards: oneOf: - - type: number - - $ref: '#/components/schemas/WaitForActiveShardOptions' + - title: count + type: integer + - title: option + $ref: '#/components/schemas/WaitForActiveShardOptions' WaitForActiveShardOptions: type: string enum: From 56081a4f1617caabf02112341a4de9de2f18bb5e Mon Sep 17 00:00:00 2001 From: "Theo N. Truong" Date: Mon, 21 Oct 2024 13:39:31 -0600 Subject: [PATCH 52/73] Made NestedAggregate inherit from AggregateBase instead of SingleBucketAggregateBase to resolve `doc_count` requirement conflict (#632) Signed-off-by: Theo Truong --- spec/schemas/_common.aggregations.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/schemas/_common.aggregations.yaml b/spec/schemas/_common.aggregations.yaml index be7ef28a4..2e64a19a6 100644 --- a/spec/schemas/_common.aggregations.yaml +++ b/spec/schemas/_common.aggregations.yaml @@ -784,7 +784,7 @@ components: - doc_count NestedAggregate: allOf: - - $ref: '#/components/schemas/SingleBucketAggregateBase' + - $ref: '#/components/schemas/AggregateBase' - type: object properties: doc_count: From 352e62019092be3f2304f2e3a500ab5441860cc4 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 23 Oct 2024 01:36:52 +1300 Subject: [PATCH 53/73] Remove unsupported runtime fields types (#634) * Remove unsupported runtime fields types Signed-off-by: Thomas Farr * Update changelog Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 4 -- spec/namespaces/indices.yaml | 2 - spec/schemas/_common.mapping.yaml | 51 -------------- spec/schemas/_core.msearch.yaml | 2 - spec/schemas/_core.reindex.yaml | 2 - spec/schemas/asynchronous_search._common.yaml | 2 - spec/schemas/cluster.stats.yaml | 66 ------------------- 8 files changed, 1 insertion(+), 129 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ec8c1b80..3f8d6843c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -134,6 +134,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358)) - Removed `shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Removed unsupported DataStream Lifecycle types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) +- Removed unsupported runtime field types ([#634](https://github.com/opensearch-project/opensearch-api-specification/pull/634)) ### Fixed diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index d96136a14..af76fa7e3 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2349,8 +2349,6 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/Fields' index_filter: $ref: '../schemas/_common.query_dsl.yaml#/components/schemas/QueryContainer' - runtime_mappings: - $ref: '../schemas/_common.mapping.yaml#/components/schemas/RuntimeFields' description: An index filter specified with the Query DSL index: content: @@ -2647,8 +2645,6 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/Fields' pit: $ref: '../schemas/_core.search.yaml#/components/schemas/PointInTimeReference' - runtime_mappings: - $ref: '../schemas/_common.mapping.yaml#/components/schemas/RuntimeFields' stats: description: |- Stats groups to associate with the search. diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index ad4a4c8dd..cbe075a6d 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2103,8 +2103,6 @@ components: $ref: '../schemas/_common.mapping.yaml#/components/schemas/RoutingField' _source: $ref: '../schemas/_common.mapping.yaml#/components/schemas/SourceField' - runtime: - $ref: '../schemas/_common.mapping.yaml#/components/schemas/RuntimeFields' description: The mapping definition required: true indices.put_settings: diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index c08585de3..2ec2dfbda 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -6,53 +6,6 @@ info: paths: {} components: schemas: - RuntimeFields: - type: object - additionalProperties: - $ref: '#/components/schemas/RuntimeField' - RuntimeField: - type: object - properties: - fetch_fields: - description: For type `lookup` - type: array - items: - $ref: '#/components/schemas/RuntimeFieldFetchFields' - format: - description: A custom format for `date` type runtime fields. - type: string - input_field: - $ref: '_common.yaml#/components/schemas/Field' - target_field: - $ref: '_common.yaml#/components/schemas/Field' - target_index: - $ref: '_common.yaml#/components/schemas/IndexName' - script: - $ref: '_common.yaml#/components/schemas/Script' - type: - $ref: '#/components/schemas/RuntimeFieldType' - required: - - type - RuntimeFieldFetchFields: - type: object - properties: - field: - $ref: '_common.yaml#/components/schemas/Field' - format: - type: string - required: - - field - RuntimeFieldType: - type: string - enum: - - boolean - - date - - double - - geo_point - - ip - - keyword - - long - - lookup TypeMapping: type: object properties: @@ -90,10 +43,6 @@ components: $ref: '#/components/schemas/SizeField' _source: $ref: '#/components/schemas/SourceField' - runtime: - type: object - additionalProperties: - $ref: '#/components/schemas/RuntimeField' enabled: type: boolean _data_stream_timestamp: diff --git a/spec/schemas/_core.msearch.yaml b/spec/schemas/_core.msearch.yaml index 19e8c6c9a..04405f280 100644 --- a/spec/schemas/_core.msearch.yaml +++ b/spec/schemas/_core.msearch.yaml @@ -152,8 +152,6 @@ components: version: description: When `true`, returns the document version as part of the hit. type: boolean - runtime_mappings: - $ref: '_common.mapping.yaml#/components/schemas/RuntimeFields' seq_no_primary_term: description: |- When `true`, returns the sequence number and primary term of the last modification. diff --git a/spec/schemas/_core.reindex.yaml b/spec/schemas/_core.reindex.yaml index 481f192ec..fc4a1ae2e 100644 --- a/spec/schemas/_core.reindex.yaml +++ b/spec/schemas/_core.reindex.yaml @@ -42,8 +42,6 @@ components: $ref: '_common.yaml#/components/schemas/Sort' _source: $ref: '_common.yaml#/components/schemas/Fields' - runtime_mappings: - $ref: '_common.mapping.yaml#/components/schemas/RuntimeFields' required: - index RemoteSource: diff --git a/spec/schemas/asynchronous_search._common.yaml b/spec/schemas/asynchronous_search._common.yaml index 691bf9ba9..ce677af0a 100644 --- a/spec/schemas/asynchronous_search._common.yaml +++ b/spec/schemas/asynchronous_search._common.yaml @@ -155,8 +155,6 @@ components: $ref: '_common.yaml#/components/schemas/Fields' pit: $ref: '_core.search.yaml#/components/schemas/PointInTimeReference' - runtime_mappings: - $ref: '_common.mapping.yaml#/components/schemas/RuntimeFields' stats: description: |- Stats groups to associate with the search. diff --git a/spec/schemas/cluster.stats.yaml b/spec/schemas/cluster.stats.yaml index cd6ff6864..55a95cd61 100644 --- a/spec/schemas/cluster.stats.yaml +++ b/spec/schemas/cluster.stats.yaml @@ -201,11 +201,6 @@ components: type: array items: $ref: '#/components/schemas/FieldTypes' - runtime_field_types: - description: Contains statistics about runtime field data types used in selected nodes. - type: array - items: - $ref: '#/components/schemas/RuntimeFieldTypes' total_field_count: description: Total number of fields in all non-system indices. type: number @@ -219,67 +214,6 @@ components: $ref: '_common.yaml#/components/schemas/ByteCount' required: - field_types - RuntimeFieldTypes: - type: object - properties: - chars_max: - description: Maximum number of characters for a single runtime field script. - type: number - chars_total: - description: Total number of characters for the scripts that define the current runtime field data type. - type: number - count: - description: Number of runtime fields mapped to the field data type in selected nodes. - type: number - doc_max: - description: Maximum number of accesses to doc_values for a single runtime field script - type: number - doc_total: - description: Total number of accesses to doc_values for the scripts that define the current runtime field data type. - type: number - index_count: - description: Number of indices containing a mapping of the runtime field data type in selected nodes. - type: number - lang: - description: Script languages used for the runtime fields scripts. - type: array - items: - type: string - lines_max: - description: Maximum number of lines for a single runtime field script. - type: number - lines_total: - description: Total number of lines for the scripts that define the current runtime field data type. - type: number - name: - $ref: '_common.yaml#/components/schemas/Name' - scriptless_count: - description: Number of runtime fields that don't declare a script. - type: number - shadowed_count: - description: Number of runtime fields that shadow an indexed field. - type: number - source_max: - description: Maximum number of accesses to _source for a single runtime field script. - type: number - source_total: - description: Total number of accesses to _source for the scripts that define the current runtime field data type. - type: number - required: - - chars_max - - chars_total - - count - - doc_max - - doc_total - - index_count - - lang - - lines_max - - lines_total - - name - - scriptless_count - - shadowed_count - - source_max - - source_total IndicesVersions: type: object properties: From 8255f1b3761a9d9653475aee18dee6acd904cf71 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 22 Oct 2024 12:40:31 -0400 Subject: [PATCH 54/73] Added a release workflow triggered on a tag. (#635) --- .cspell | 4 ++ .github/workflows/release.yml | 99 +++++++++++++++++++++++++++++++++++ CHANGELOG.md | 1 + RELEASING.md | 28 ++++++++++ spec/_info.yaml | 2 +- 5 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/release.yml create mode 100644 RELEASING.md diff --git a/.cspell b/.cspell index 3d9ace5d3..005f91cd5 100644 --- a/.cspell +++ b/.cspell @@ -106,6 +106,7 @@ Nanos nbest nfkc ngram +nguyenvukhang niofs nmslib nodeattrs @@ -158,6 +159,7 @@ Rudnick ruleset scriptless securityconfig +signoff slowlog Slowlog slurpfile @@ -165,6 +167,7 @@ smartcn Smartcn snapshotted softmax +softprops Sorani sslinfo stoptags @@ -181,6 +184,7 @@ tenantinfo termvectors tfidf Tfidf +thomaseizinger Tika tlbr tokenfilters diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..8103a9e5c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,99 @@ +name: Draft a Release + +on: + push: + tags: + - 'v*.*.*' + +jobs: + draft-a-release: + name: Draft a Release + runs-on: ubuntu-latest + permissions: write-all + + steps: + - name: Checkout the repo + uses: actions/checkout@v4 + + - name: Get Current Version + id: current + run: | + echo version=$(yq .version spec/_info.yaml) >> "$GITHUB_OUTPUT" + + - name: Check Version + run: | + echo current=v${{ steps.current.outputs.version }} + echo tag=${{ github.ref_name }} + if "v${{ steps.current.outputs.version }}" != "${{ github.ref_name }}"; then + echo "::error::Failing, reason: version ${{ steps.current.outputs.version }} does not match tag ${{ github.ref_name }}." + exit 1 + else + echo "::notice::Found version ${{ steps.current.outputs.version }}, all good." + fi + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: Build + run: npm ci && npm run merge + + - name: Upload Artifacts + id: upload + uses: actions/upload-artifact@v4 + with: + name: build + path: | + build/* + + - name: Extract Changelog + id: changelog + uses: sean0x42/markdown-extract@v2 + with: + file: CHANGELOG.md + pattern: Unreleased + no-print-matched-heading: true + + - name: Draft a release + uses: softprops/action-gh-release@v2 + id: release + with: + draft: true + generate_release_notes: false + tag_name: ${{ github.ref_name }} + body: | + ${{ steps.changelog.outputs.markdown }} + + ### Links + - [Build Artifact](${{ steps.upload.outputs.artifact-url }}) + files: | + LICENSE.txt + build/* + + - name: Increment Version + uses: nguyenvukhang/semver-increment@v1 + id: version + with: + increment: patch + version-file: spec/_info.yaml + version-regex: '^version: (.*)' + + - name: Update CHANGELOG to the Next Developer Iteration + uses: thomaseizinger/keep-a-changelog-new-release@v3 + with: + tag: ${{ github.ref_name }} + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'Preparing for next developer iteration ${{ steps.version.outputs.version }}.' + signoff: true + branch: prepare-v-next + base: main + delete-branch: true + title: 'Preparing for next developer iteration, ${{ steps.version.outputs.version }}.' + body: | + I've drafted a release, [${{ github.ref_name }}](${{ steps.release.outputs.url }}), please verify and release it. + Preparing for the next developer iteration, ${{ steps.version.outputs.version }}. diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f8d6843c..c6cee833d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -102,6 +102,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `404` response for `DELETE /{index}`, `GET /{index}/_doc/{id}`, `DELETE /{index}/_doc/{id}` ([#589](https://github.com/opensearch-project/opensearch-api-specification/pull/589)) - Added ability to pass `InlineScript` as a simple string ([#605](https://github.com/opensearch-project/opensearch-api-specification/pull/605)) - Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) +- Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) ### Changed diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 000000000..83a58e3a7 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,28 @@ +- [Overview](#overview) +- [Branching](#branching) + - [Release Branching](#release-branching) + - [Feature Branches](#feature-branches) +- [Release Labels](#release-labels) +- [Releasing](#releasing) + +## Overview + +This document explains the release strategy for opensearch-api-specification. + +## Versioning + +At this time this project follows minor semantic versioning, i.e. a breaking change is an increment from 0.1.0 to 0.2.0 and a non-breaking change is an increment from 0.1.0 to 0.1.1. A version 1.0 of the API specification will be released when the spec is known to be complete against any current version of OpenSearch. + +## Branching + +Given the early release stages of this repo we only maintain a **main** branch where all merges take place and code moves fast. + +# Releasing + +A release can be done by any [maintainer](MAINTAINERS.md). + +1. Check out the [upstream repo](https://github.com/opensearch-project/opensearch-api-specification) and ensure the repo is up-to-date with `git pull origin main` and that you do not have any local changes. +2. Create a tag, e.g. `git tag v0.1.0`, and push it to GitHub with `git push origin --tags`. +3. The [release workflow](.github/workflows/release.yml) will be automatically kicked off, a draft release, and a pull request that increments the version in [spec/_info.yaml](spec/_info.yaml) will be created. +4. Verify and release the draft. +5. Approve, and merge the version increment pull request. diff --git a/spec/_info.yaml b/spec/_info.yaml index d8d63d088..58818da48 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -1,5 +1,5 @@ $schema: ./json_schemas/_info.schema.yaml title: OpenSearch API Specification -version: 1.0.0 +version: 0.1.0 x-api-version: 2.16.0 From abc7163185733c1008fb7c28117cf36c051f3292 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Wed, 23 Oct 2024 05:12:56 -0700 Subject: [PATCH 55/73] Query Insights API spec (#625) * Query Insights API spec Signed-off-by: Chenyang Ji * rename to insights to follow the api pattern Signed-off-by: Chenyang Ji * add changelog Signed-off-by: Chenyang Ji * add CI/CD matrix Signed-off-by: Chenyang Ji --------- Signed-off-by: Chenyang Ji --- .github/workflows/test-spec.yml | 2 + CHANGELOG.md | 1 + spec/namespaces/insights.yaml | 34 +++ spec/schemas/insights._common.yaml | 228 ++++++++++++++++++ .../plugins/query_insights/docker-compose.yml | 15 ++ .../query_insights/insights/top_queries.yaml | 99 ++++++++ 6 files changed, 379 insertions(+) create mode 100644 spec/namespaces/insights.yaml create mode 100644 spec/schemas/insights._common.yaml create mode 100644 tests/plugins/query_insights/docker-compose.yml create mode 100644 tests/plugins/query_insights/insights/top_queries.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index a4c354c89..121632390 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -40,6 +40,8 @@ jobs: tests: plugins/streaming - version: 2.17.0 tests: plugins/notifications + - version: 2.17.0 + tests: plugins/query_insights - version: 2.18.0 hub: opensearchstaging ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' diff --git a/CHANGELOG.md b/CHANGELOG.md index c6cee833d..f291cd4b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added ability to pass `InlineScript` as a simple string ([#605](https://github.com/opensearch-project/opensearch-api-specification/pull/605)) - Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) +- Added API spec for query insights plugin ([#625](https://github.com/opensearch-project/opensearch-api-specification/pull/625)) ### Changed diff --git a/spec/namespaces/insights.yaml b/spec/namespaces/insights.yaml new file mode 100644 index 000000000..a9b75f3c9 --- /dev/null +++ b/spec/namespaces/insights.yaml @@ -0,0 +1,34 @@ +openapi: 3.1.0 +info: + title: Query Insights API + description: API to retrieve top queries based on latency, CPU, or memory usage. + version: 1.0.0 +paths: + /_insights/top_queries: + get: + operationId: insights.top_queries.0 + x-operation-group: insights.top_queries + x-version-added: '1.0' + description: Retrieves the top queries based on the given metric type (latency, CPU, or memory). + parameters: + - $ref: '#/components/parameters/insights.top_queries::query.type' + responses: + '200': + $ref: '#/components/responses/insights.top_queries@200' + +components: + parameters: + insights.top_queries::query.type: + name: type + in: query + required: true + description: Get top n queries by a specific metric. + schema: + type: string + enum: [cpu, latency, memory] + responses: + insights.top_queries@200: + content: + application/json: + schema: + $ref: '../schemas/insights._common.yaml#/components/schemas/TopQueriesResponse' diff --git a/spec/schemas/insights._common.yaml b/spec/schemas/insights._common.yaml new file mode 100644 index 000000000..b4a700360 --- /dev/null +++ b/spec/schemas/insights._common.yaml @@ -0,0 +1,228 @@ +openapi: 3.1.0 +info: + title: Schemas of query insights + description: Schemas of query insights + version: 1.0.0 +paths: {} +components: + schemas: + TopQueriesResponse: + type: object + properties: + top_queries: + type: array + items: + type: object + $ref: '#/components/schemas/TopQuery' + required: + - top_queries + TopQuery: + type: object + properties: + timestamp: + type: integer + description: The timestamp of the query execution. + total_shards: + type: integer + description: The total number of shards involved in the query. + task_resource_usages: + type: array + items: + type: object + $ref: '#/components/schemas/TaskResourceUsages' + # eslint-disable-next-line @cspell/spellchecker + query_hashcode: + type: string + description: The hash code of the query. + labels: + type: object + description: Additional labels for the query. + search_type: + type: string + description: The search query type (e.g., query_then_fetch). + source: + type: object + $ref: '#/components/schemas/Source' + node_id: + type: string + description: The node ID associated with the query. + indices: + type: array + items: + type: string + description: The indices involved in the query. + phase_latency_map: + type: object + measurements: + type: object + $ref: '#/components/schemas/Measurements' + TaskResourceUsages: + type: object + properties: + action: + type: string + description: The action type of the task. + taskId: + type: integer + description: The task ID. + parentTaskId: + type: integer + description: The parent task ID. + nodeId: + type: string + description: The node ID where the task was executed. + taskResourceUsage: + type: object + $ref: '#/components/schemas/TaskResourceUsage' + TaskResourceUsage: + type: object + properties: + cpu_time_in_nanos: + type: integer + description: The CPU time used in nanoseconds. + memory_in_bytes: + type: integer + description: The memory usage in bytes. + Source: + type: object + properties: + aggregations: + description: Defines the aggregations that are run as part of the search request. + type: object + collapse: + $ref: '_core.search.yaml#/components/schemas/FieldCollapse' + explain: + description: If true, returns detailed information about score computation as part of a hit. + type: boolean + ext: + description: Configuration of search extensions defined by OpenSearch plugins. + type: object + additionalProperties: + type: object + from: + description: |- + Starting document offset. + Needs to be non-negative. + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. + To page through more hits, use the `search_after` parameter. + type: number + highlight: + $ref: '_core.search.yaml#/components/schemas/Highlight' + track_total_hits: + $ref: '_core.search.yaml#/components/schemas/TrackHits' + indices_boost: + description: Boosts the _score of documents from specified indices. + type: array + items: + type: object + additionalProperties: + type: number + docvalue_fields: + description: |- + Array of wildcard (`*`) patterns. + The request returns doc values for field names matching these patterns in the `hits.fields` property of the response. + type: array + items: + $ref: '_common.query_dsl.yaml#/components/schemas/FieldAndFormat' + min_score: + description: |- + Minimum `_score` for matching documents. + Documents with a lower `_score` are not included in the search results. + type: number + post_filter: + $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' + profile: + description: |- + Set to `true` to return detailed timing information about the execution of individual components in a search request. + NOTE: This is a debugging tool and adds significant overhead to search execution. + type: boolean + query: + $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' + script_fields: + description: Retrieve a script evaluation (based on different fields) for each hit. + type: object + additionalProperties: + $ref: '_common.yaml#/components/schemas/ScriptField' + search_after: + $ref: '_common.yaml#/components/schemas/SortResults' + size: + description: |- + The number of hits to return. + By default, you cannot page through more than 10,000 hits using the `from` and `size` parameters. + To page through more hits, use the `search_after` parameter. + type: number + slice: + $ref: '_common.yaml#/components/schemas/SlicedScroll' + sort: + $ref: '_common.yaml#/components/schemas/Sort' + _source: + $ref: '_core.search.yaml#/components/schemas/SourceConfig' + fields: + description: |- + Array of wildcard (`*`) patterns. + The request returns values for field names matching these patterns in the `hits.fields` property of the response. + type: array + items: + $ref: '_common.query_dsl.yaml#/components/schemas/FieldAndFormat' + suggest: + $ref: '_core.search.yaml#/components/schemas/Suggester' + terminate_after: + description: |- + Maximum number of documents to collect for each shard. + If a query reaches this limit, OpenSearch terminates the query early. + OpenSearch collects documents before sorting. + Use with caution. + OpenSearch applies this parameter to each shard handling the request. + When possible, let OpenSearch perform early termination automatically. + Avoid specifying this parameter for requests that target data streams with backing indices across multiple data tiers. + If set to `0` (default), the query does not terminate early. + type: integer + format: int32 + timeout: + description: |- + Specifies the period of time to wait for a response from each shard. + If no response is received before the timeout expires, the request fails and returns an error. + Defaults to no timeout. + type: string + track_scores: + description: If true, calculate and return document scores, even if the scores are not used for sorting. + type: boolean + version: + description: If true, returns document version as part of a hit. + type: boolean + seq_no_primary_term: + description: If `true`, returns sequence number and primary term of the last modification of each hit. + type: boolean + stored_fields: + $ref: '_common.yaml#/components/schemas/Fields' + pit: + $ref: '_core.search.yaml#/components/schemas/PointInTimeReference' + stats: + description: |- + Stats groups to associate with the search. + Each group maintains a statistics aggregation for its associated searches. + You can retrieve these stats using the indices stats API. + type: array + items: + type: string + Measurement: + type: object + properties: + number: + type: integer + count: + type: integer + aggregationType: + type: string + Measurements: + type: object + properties: + latency: + type: object + $ref: '#/components/schemas/Measurement' + cpu: + type: object + $ref: '#/components/schemas/Measurement' + memory: + type: object + $ref: '#/components/schemas/Measurement' \ No newline at end of file diff --git a/tests/plugins/query_insights/docker-compose.yml b/tests/plugins/query_insights/docker-compose.yml new file mode 100644 index 000000000..69c7d01fe --- /dev/null +++ b/tests/plugins/query_insights/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node + - search.insights.top_queries.cpu.enabled=true + - search.insights.top_queries.latency.enabled=true + - search.insights.top_queries.memory.enabled=true diff --git a/tests/plugins/query_insights/insights/top_queries.yaml b/tests/plugins/query_insights/insights/top_queries.yaml new file mode 100644 index 000000000..d19828912 --- /dev/null +++ b/tests/plugins/query_insights/insights/top_queries.yaml @@ -0,0 +1,99 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test top n queries API in the Query Insights plugin. +version: '>= 2.15' +prologues: + - path: /movies + method: PUT + request: + payload: + mappings: + properties: + director: + type: text + fielddata: true + fields: + raw: + type: keyword + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball} + - {create: {_index: movies}} + - {director: Bennett Miller, title: The Cruise} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive} + + - path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + size: 0 + aggregations: + directors: + terms: + field: director.raw + +chapters: + - synopsis: Retrieve default top queries. + path: /_insights/top_queries + retry: + count: 2 + wait: 5000 + method: GET + response: + status: 200 + content_type: application/json + payload: + top_queries: [] + - synopsis: Retrieve top queries by latency. + path: /_insights/top_queries + retry: + count: 2 + wait: 5000 + parameters: + type: latency + method: GET + response: + status: 200 + content_type: application/json + payload: + top_queries: [] + - synopsis: Retrieve top queries by cpu usage. + path: /_insights/top_queries + retry: + count: 2 + wait: 5000 + parameters: + type: cpu + method: GET + response: + status: 200 + content_type: application/json + payload: + top_queries: [] + - synopsis: Retrieve top queries by memory usage. + path: /_insights/top_queries + retry: + count: 2 + wait: 5000 + parameters: + type: memory + method: GET + response: + status: 200 + content_type: application/json + payload: + top_queries: [] + +epilogues: + - path: /movies + method: DELETE + status: [200, 404] From 54f0af9bcedfa8bd8f5dc225becd0941d6184697 Mon Sep 17 00:00:00 2001 From: Chenyang Ji Date: Thu, 24 Oct 2024 12:58:54 -0700 Subject: [PATCH 56/73] Add adjust_pure_negative for bool queries (#641) * allow additional properties for source field Signed-off-by: Chenyang Ji * add adjust_pure_negative api spec Signed-off-by: Chenyang Ji --------- Signed-off-by: Chenyang Ji --- CHANGELOG.md | 1 + spec/schemas/_common.query_dsl.yaml | 7 +++++++ .../query_insights/insights/top_queries.yaml | 13 +++++++++++++ 3 files changed, 21 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f291cd4b9..fcb22f02d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) - Added API spec for query insights plugin ([#625](https://github.com/opensearch-project/opensearch-api-specification/pull/625)) +- Added API spec for `adjust_pure_negative` for bool queries ([#641](https://github.com/opensearch-project/opensearch-api-specification/pull/641)) ### Changed diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 070d84ce8..b8e43a867 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -270,6 +270,13 @@ components: - type: array items: $ref: '#/components/schemas/QueryContainer' + adjust_pure_negative: + description: |- + Ensures correct behavior when a query contains only must_not clauses. + By default set to true, OpenSearch adds a match-all clause to ensure results are returned from Lucene, with the must_not conditions applied as filters. + If set to false, the query may return no results, as Lucene typically requires at least one positive condition. + type: boolean + default: true QueryBase: type: object properties: diff --git a/tests/plugins/query_insights/insights/top_queries.yaml b/tests/plugins/query_insights/insights/top_queries.yaml index d19828912..d9a4c156e 100644 --- a/tests/plugins/query_insights/insights/top_queries.yaml +++ b/tests/plugins/query_insights/insights/top_queries.yaml @@ -41,6 +41,19 @@ prologues: terms: field: director.raw + - path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + query: + bool: + adjust_pure_negative: true + must: + match: + title: Drive + chapters: - synopsis: Retrieve default top queries. path: /_insights/top_queries From 7bf34a41c78a51433f628b54588d094afdbdb345 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 16:52:01 -0400 Subject: [PATCH 57/73] Documented docker container testing. (#642) Signed-off-by: dblock --- TESTING_GUIDE.md | 131 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 108 insertions(+), 23 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 65312657e..ea3867c1a 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,27 +1,31 @@ -* [Spec Testing Guide](#spec-testing-guide) - * [Running Spec Tests](#running-spec-tests) - * [Running Spec Tests Locally](#running-spec-tests-locally) - * [Prerequisites](#prerequisites-) - * [OpenSearch Cluster](#opensearch-cluster-) - * [Run Tests](#run-tests) - * [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) - * [Common Errors](#common-errors) - * [401 Unauthorized](#401-unauthorized) - * [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) - * [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) - * [Writing Spec Tests](#writing-spec-tests) - * [Simple Test Story](#simple-test-story) - * [Using Output from Previous Chapters](#using-output-from-previous-chapters) - * [Managing Versions](#managing-versions) - * [Managing Distributions](#managing-distributions) - * [Waiting for Tasks](#waiting-for-tasks) - * [Warnings](#warnings) - * [multiple-paths-detected](#multiple-paths-detected) - * [Suppressing Warnings](#suppressing-warnings) - * [Collecting Test Coverage](#collecting-test-coverage) - * [Coverage Summary](#coverage-summary) - * [Coverage Report](#coverage-report) +- [Spec Testing Guide](#spec-testing-guide) + - [Running Spec Tests](#running-spec-tests) + - [Running Spec Tests Locally](#running-spec-tests-locally) + - [Prerequisites](#prerequisites) + - [OpenSearch Cluster](#opensearch-cluster) + - [Run Tests](#run-tests) + - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) + - [Common Errors](#common-errors) + - [401 Unauthorized](#401-unauthorized) + - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + - [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) + - [Writing Spec Tests](#writing-spec-tests) + - [Simple Test Story](#simple-test-story) + - [Using Output from Previous Chapters](#using-output-from-previous-chapters) + - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) + - [Waiting for Tasks](#waiting-for-tasks) + - [Warnings](#warnings) + - [multiple-paths-detected](#multiple-paths-detected) + - [Suppressing Warnings](#suppressing-warnings) + - [Collecting Test Coverage](#collecting-test-coverage) + - [Coverage Summary](#coverage-summary) + - [Coverage Report](#coverage-report) + - [Integration Testing](#integration-testing) + - [Stable Releases](#stable-releases) + - [Custom Setup](#custom-setup) + - [Future Releases](#future-releases) # Spec Testing Guide @@ -375,4 +379,85 @@ The test tool can display detailed and hierarchal test coverage with `--coverage GET /_alias/{name} POST /_alias/{name} HEAD /_alias/{name} +``` + +## Integration Testing + +This project runs integration tests against multiple versions of OpenSearch in the [test-spec.yml](.github/workflows/test-spec.yml) workflow. + +### Stable Releases + +The simplest entry in the test matrix executes tests in [tests/default](tests/default) against a released version of OpenSearch. +For example, the following entries run tests gainst OpenSearch 1.3.17 and 2.17.0. + +```yaml +entry: + - version: 1.3.17 + - version: 2.17.0 +``` + +### Custom Setup + +Some tests require a custom docker image. For example, testing the notifications plugin requires a custom webhook. This can be done by creating a custom `docker-compose.yml`, such as the one in [tests/plugins/notifications](tests/plugins/notifications/docker-compose.yml). + +```yaml +webhook: + image: python:latest + volumes: + - ./server.py:/server.py + ports: + - '8080:8080' + entrypoint: python server.py +``` + +The following example in the test matrix will use the custom `docker-compose.yml` and execute all tests in [tests/plugins/notifications](tests/plugins/notifications). + +```yaml +entry: + - version: 2.17.0 + tests: plugins/notifications +``` + +### Future Releases + +Snapshot builds of OpenSearch are available on Docker Hub under [opensearchstaging/opensearch/tags](https://hub.docker.com/r/opensearchstaging/opensearch/tags). + +The following example in the test matrix will use [a snapshot build of OpenSearch 2.18](https://hub.docker.com/layers/opensearchstaging/opensearch/2.18.0/images/sha256-504a9c42bc1b13cb47b39a29db8a9d300d01b8851fb95dbb9db6770f478e45b5?context=explore) to execute the default test suite in [tests/default](tests/default/). + +```yaml +- version: 2.18.0 + hub: opensearchstaging + ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' +``` + +It's important to note that snapshot builds may not contain all plugins, and may contain previous versions of a plugin if the current code failed to build. It's therefore possible that updating a SHA to test new functionality available in a more recent build causes failures with existing tests. As of today the only workaround is to try the next build that will hopefully have more/all the plugins. For a discussion about this problem see [opensearch-build#5130](https://github.com/opensearch-project/opensearch-build/issues/5130). + +Use the following command to retrieve the manifest of a given build to make it easier to identify a SHA that includes all the plugins. + +```bash +$ docker run -it --entrypoint bash opensearchstaging/opensearch:2.18.0@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9 -c "cat /usr/share/opensearch/manifest.yml" +``` + +```yaml +--- +schema-version: '1.1' +build: + name: OpenSearch + version: 2.18.0 + platform: linux + architecture: x64 + distribution: tar + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.18.0/10320/linux/x64/tar/dist/opensearch/opensearch-2.18.0-linux-x64.tar.gz + id: '10320' +components: + - name: OpenSearch + repository: https://github.com/opensearch-project/OpenSearch.git + ref: 2.x + commit_id: b67f76541b78e58844b54305eb21232e78167744 + location: https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/2.18.0/10320/linux/x64/tar/builds/opensearch/dist/opensearch-min-2.18.0-linux-x64.tar.gz + - name: common-utils + repository: https://github.com/opensearch-project/common-utils.git + ref: 2.x + commit_id: 63ee9746ce04a8eace0ba6320e93bf5833f6a4a6 +... ``` \ No newline at end of file From 82e37830ef2324c46365bf5d53eb735c151e1a4a Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 16:52:46 -0400 Subject: [PATCH 58/73] Fix: model training intermittent test failure. (#644) Signed-off-by: dblock --- spec/namespaces/knn.yaml | 6 +++++- tests/default/knn/train_model.yaml | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index c38bba5b7..77b6a34cb 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -283,7 +283,11 @@ components: required: true responses: knn.delete_model@200: {} - knn.get_model@200: {} + knn.get_model@200: + content: + application/json: + schema: + type: object knn.search_models@200: {} knn.stats@200: {} knn.train_model@200: diff --git a/tests/default/knn/train_model.yaml b/tests/default/knn/train_model.yaml index b45ac5c6f..cb58ed97c 100644 --- a/tests/default/knn/train_model.yaml +++ b/tests/default/knn/train_model.yaml @@ -41,6 +41,9 @@ prologues: - {index: {_index: movies, _id: '9'}} - {recommendation_vector: [9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5, 9.5], duration: 8.9} status: [200] + - method: POST + path: /_refresh + status: [200] epilogues: - path: /movies method: DELETE @@ -70,4 +73,18 @@ chapters: response: status: 200 output: - test_model_id: payload.model_id \ No newline at end of file + test_model_id: payload.model_id + - synopsis: Wait for the model to get trained. + warnings: + multiple-paths-detected: false + method: GET + path: /_plugins/_knn/models/{model_id} + parameters: + model_id: ${train_model.test_model_id} + retry: + count: 3 + response: + status: 200 + payload: + model_id: ${train_model.test_model_id} + state: created From 110474fbd4eac56318fcea220ff9a6ebcd615d04 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 16:53:53 -0400 Subject: [PATCH 59/73] Updated 2.18 to RC2. (#643) * Updated 2.18 to RC2. Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 +- CHANGELOG.md | 1 + spec/schemas/nodes._common.yaml | 8 ++++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 121632390..23f0fffc6 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -44,7 +44,7 @@ jobs: tests: plugins/query_insights - version: 2.18.0 hub: opensearchstaging - ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9' + ref: '@sha256:57a1cd1142d68c203e2e4aa0666d9a691e1e409a5d14aa4a8f5044036f05cf06' - version: 3.0.0 hub: opensearchstaging ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de' diff --git a/CHANGELOG.md b/CHANGELOG.md index fcb22f02d..dbf448b1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) - Added API spec for query insights plugin ([#625](https://github.com/opensearch-project/opensearch-api-specification/pull/625)) - Added API spec for `adjust_pure_negative` for bool queries ([#641](https://github.com/opensearch-project/opensearch-api-specification/pull/641)) +- Added `remote_store` to node `Stats` ([#643](https://github.com/opensearch-project/opensearch-api-specification/pull/643)) ### Changed diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 9ba3c0621..4c071b743 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -276,12 +276,20 @@ components: $ref: '#/components/schemas/ShardSearchPipelineStats' segment_replication_backpressure: $ref: '#/components/schemas/ShardSegmentReplicationBackpressureStats' + remote_store: + $ref: '#/components/schemas/RemoteStoreStats' repositories: $ref: '#/components/schemas/ShardRepositoriesStats' admission_control: $ref: '#/components/schemas/ShardAdmissionControlStats' caches: $ref: '#/components/schemas/ShardCachesStats' + RemoteStoreStats: + type: object + properties: + last_successful_fetch_of_pinned_timestamps: + description: Timestamp for the last successful fetch of pinned timestamps. + $ref: '_common.yaml#/components/schemas/StringifiedEpochTimeUnitSeconds' ShardClusterManagerThrottlingStats: type: object properties: From 61e6ec31d497e06b6272d664afded93db8dfee62 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 16:57:50 -0400 Subject: [PATCH 60/73] Added editorial style guide checking. (#620) * Added editorial style guide checking. Signed-off-by: dblock Signed-off-by: Theo N. Truong Co-authored-by: Theo N. Truong --- .cspell | 13 ++ .../styles/OpenSearch/AcronymParentheses.yml | 75 +++++++++ .github/vale/styles/OpenSearch/Ampersand.yml | 6 + .github/vale/styles/OpenSearch/Cyber.yml | 5 + .../vale/styles/OpenSearch/DashSpacing.yml | 9 + .../styles/OpenSearch/DirectionAboveBelow.yml | 22 +++ .../styles/OpenSearch/DirectionTopBottom.yml | 16 ++ .../vale/styles/OpenSearch/Exclamation.yml | 6 + .../vale/styles/OpenSearch/FailoverNoun.yml | 7 + .../vale/styles/OpenSearch/FailoverVerb.yml | 7 + .../vale/styles/OpenSearch/FutureTense.yml | 7 + .github/vale/styles/OpenSearch/Inclusive.yml | 15 ++ .../OpenSearch/LatinismsElimination.yml | 10 ++ .../OpenSearch/LatinismsSubstitution.yml | 15 ++ .github/vale/styles/OpenSearch/LoginNoun.yml | 7 + .github/vale/styles/OpenSearch/LoginVerb.yml | 7 + .github/vale/styles/OpenSearch/LogoutNoun.yml | 7 + .github/vale/styles/OpenSearch/LogoutVerb.yml | 7 + .../vale/styles/OpenSearch/OxfordComma.yml | 7 + .../vale/styles/OpenSearch/PassiveVoice.yml | 45 +++++ .github/vale/styles/OpenSearch/Please.yml | 9 + .github/vale/styles/OpenSearch/Range.yml | 7 + .github/vale/styles/OpenSearch/Repetition.yml | 6 + .../vale/styles/OpenSearch/RolloverNoun.yml | 7 + .../vale/styles/OpenSearch/RolloverVerb.yml | 7 + .github/vale/styles/OpenSearch/SetupNoun.yml | 7 + .github/vale/styles/OpenSearch/SetupVerb.yml | 7 + .../vale/styles/OpenSearch/SignatureV4.yml | 11 ++ .github/vale/styles/OpenSearch/Simple.yml | 10 ++ .../styles/OpenSearch/SpacingPunctuation.yml | 9 + .../vale/styles/OpenSearch/SpacingSlash.yml | 6 + .../vale/styles/OpenSearch/SpacingWords.yml | 6 + .github/vale/styles/OpenSearch/Spelling.yml | 5 + .../styles/OpenSearch/SubstitutionsError.yml | 52 ++++++ .../OpenSearch/SubstitutionsSuggestion.yml | 16 ++ .../vale/styles/OpenSearch/TimeoutNoun.yml | 7 + .../vale/styles/OpenSearch/TimeoutVerb.yml | 7 + .github/vale/styles/OpenSearch/UnitsNames.yml | 12 ++ .../vale/styles/OpenSearch/UnitsSpacing.yml | 8 + .github/vale/styles/OpenSearch/Version.yml | 6 + .../OpenSearch/acronyms/accept.txt | 1 + .../OpenSearch/acronyms/reject.txt | 1 + .../OpenSearch/plugins/accept.txt | 30 ++++ .../OpenSearch/plugins/reject.txt | 1 + .../OpenSearch/products/accept.txt | 100 +++++++++++ .../OpenSearch/products/reject.txt | 1 + .../vocabularies/OpenSearch/words/accept.txt | 155 ++++++++++++++++++ .../vocabularies/OpenSearch/words/reject.txt | 1 + .github/workflows/lint.yml | 4 +- .github/workflows/style.yml | 31 ++++ .vale.ini | 49 ++++++ CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 93 ++++++----- package-lock.json | 1 + package.json | 4 +- .../src/prepare-for-vale/KeepDescriptions.ts | 55 +++++++ .../src/prepare-for-vale/prepare-for-vale.ts | 27 +++ .../prepare-for-vale/KeepDescriptions.test.ts | 43 +++++ .../tests/prepare-for-vale/fixtures/spec.txt | 26 +++ .../tests/prepare-for-vale/fixtures/spec.yaml | 26 +++ .../prepare-for-vale/prepare-for-vale.test.ts | 22 +++ 61 files changed, 1125 insertions(+), 45 deletions(-) create mode 100644 .github/vale/styles/OpenSearch/AcronymParentheses.yml create mode 100644 .github/vale/styles/OpenSearch/Ampersand.yml create mode 100644 .github/vale/styles/OpenSearch/Cyber.yml create mode 100644 .github/vale/styles/OpenSearch/DashSpacing.yml create mode 100644 .github/vale/styles/OpenSearch/DirectionAboveBelow.yml create mode 100644 .github/vale/styles/OpenSearch/DirectionTopBottom.yml create mode 100644 .github/vale/styles/OpenSearch/Exclamation.yml create mode 100644 .github/vale/styles/OpenSearch/FailoverNoun.yml create mode 100644 .github/vale/styles/OpenSearch/FailoverVerb.yml create mode 100644 .github/vale/styles/OpenSearch/FutureTense.yml create mode 100644 .github/vale/styles/OpenSearch/Inclusive.yml create mode 100644 .github/vale/styles/OpenSearch/LatinismsElimination.yml create mode 100644 .github/vale/styles/OpenSearch/LatinismsSubstitution.yml create mode 100644 .github/vale/styles/OpenSearch/LoginNoun.yml create mode 100644 .github/vale/styles/OpenSearch/LoginVerb.yml create mode 100644 .github/vale/styles/OpenSearch/LogoutNoun.yml create mode 100644 .github/vale/styles/OpenSearch/LogoutVerb.yml create mode 100644 .github/vale/styles/OpenSearch/OxfordComma.yml create mode 100644 .github/vale/styles/OpenSearch/PassiveVoice.yml create mode 100644 .github/vale/styles/OpenSearch/Please.yml create mode 100644 .github/vale/styles/OpenSearch/Range.yml create mode 100644 .github/vale/styles/OpenSearch/Repetition.yml create mode 100644 .github/vale/styles/OpenSearch/RolloverNoun.yml create mode 100644 .github/vale/styles/OpenSearch/RolloverVerb.yml create mode 100644 .github/vale/styles/OpenSearch/SetupNoun.yml create mode 100644 .github/vale/styles/OpenSearch/SetupVerb.yml create mode 100644 .github/vale/styles/OpenSearch/SignatureV4.yml create mode 100644 .github/vale/styles/OpenSearch/Simple.yml create mode 100644 .github/vale/styles/OpenSearch/SpacingPunctuation.yml create mode 100644 .github/vale/styles/OpenSearch/SpacingSlash.yml create mode 100644 .github/vale/styles/OpenSearch/SpacingWords.yml create mode 100644 .github/vale/styles/OpenSearch/Spelling.yml create mode 100644 .github/vale/styles/OpenSearch/SubstitutionsError.yml create mode 100644 .github/vale/styles/OpenSearch/SubstitutionsSuggestion.yml create mode 100644 .github/vale/styles/OpenSearch/TimeoutNoun.yml create mode 100644 .github/vale/styles/OpenSearch/TimeoutVerb.yml create mode 100644 .github/vale/styles/OpenSearch/UnitsNames.yml create mode 100644 .github/vale/styles/OpenSearch/UnitsSpacing.yml create mode 100644 .github/vale/styles/OpenSearch/Version.yml create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/acronyms/accept.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/acronyms/reject.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/plugins/accept.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/plugins/reject.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/products/accept.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/products/reject.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/words/accept.txt create mode 100644 .github/vale/styles/config/vocabularies/OpenSearch/words/reject.txt create mode 100644 .github/workflows/style.yml create mode 100644 .vale.ini create mode 100644 tools/src/prepare-for-vale/KeepDescriptions.ts create mode 100644 tools/src/prepare-for-vale/prepare-for-vale.ts create mode 100644 tools/tests/prepare-for-vale/KeepDescriptions.test.ts create mode 100644 tools/tests/prepare-for-vale/fixtures/spec.txt create mode 100644 tools/tests/prepare-for-vale/fixtures/spec.yaml create mode 100644 tools/tests/prepare-for-vale/prepare-for-vale.test.ts diff --git a/.cspell b/.cspell index 005f91cd5..274d2da09 100644 --- a/.cspell +++ b/.cspell @@ -51,10 +51,14 @@ fragmenter Fragmenter freqs generateonbehalfoftoken +geodistance geohash geohex Geohex geoip +geopoint +geoshape +geospatial geotile gsub Gsub @@ -65,6 +69,7 @@ homoscedastic hotthreads huggingface hybridfs +ignorecase integ internalusers Intertransport @@ -111,6 +116,7 @@ niofs nmslib nodeattrs nodesdn +nonword noops nori Nori @@ -128,7 +134,11 @@ Oversample performanceanalyzer permissionsinfo pipefail +preconfigure +preconfigured prefilter +pretrain +pretrained prirep rawfile readingform @@ -186,11 +196,13 @@ tfidf Tfidf thomaseizinger Tika +timeframe tlbr tokenfilters translog Translog trbl +truststore tubone Undeploys unigrams @@ -203,6 +215,7 @@ urldecode vectory whoamiprotected wordnet +yber Yrtsd جامد جامدات \ No newline at end of file diff --git a/.github/vale/styles/OpenSearch/AcronymParentheses.yml b/.github/vale/styles/OpenSearch/AcronymParentheses.yml new file mode 100644 index 000000000..a3deecc01 --- /dev/null +++ b/.github/vale/styles/OpenSearch/AcronymParentheses.yml @@ -0,0 +1,75 @@ +extends: conditional +message: "'%s': Spell out acronyms the first time that you use them on a page and follow them with the acronym in parentheses. Subsequently, use the acronym alone." +link: 'https://github.com/opensearch-project/documentation-website/blob/main/STYLE_GUIDE.md#acronyms' +level: warning +scope: summary +ignorecase: false +# Ensures that the existence of 'first' implies the existence of 'second'. +first: '\b((? -* [Developer Guide](#developer-guide) - * [Getting Started](#getting-started) - * [Specification](#specification) - * [File Structure](#file-structure) - * [Grouping Operations](#grouping-operations) - * [Grouping Schemas](#grouping-schemas) - * [Superseded Operations](#superseded-operations) - * [Global Parameters](#global-parameters) - * [OpenAPI Extensions](#openapi-extensions) - * [Writing Spec Tests](#writing-spec-tests) - * [Tools](#tools) - * [Setup](#setup) - * [Spec Merger](#spec-merger) - * [Arguments](#arguments) - * [Example](#example) - * [Spec Linter](#spec-linter) - * [Arguments](#arguments-1) - * [Example](#example-1) - * [Spec Tester](#spec-tester) - * [Dump Cluster Spec](#dump-cluster-spec) - * [Arguments](#arguments-2) - * [Example](#example-2) - * [Coverage](#coverage) - * [Arguments](#arguments-3) - * [Example](#example-3) - * [Tools Testing](#tools-testing) - * [Tools Linting](#tools-linting) - * [Workflows](#workflows) - * [Analyze PR Changes](#analyze-pr-changes) - * [Build](#build) - * [Deploy GitHub Pages](#deploy-github-pages) - * [Comment on PR](#comment-on-pr) - * [Test Tools (Unit)](#test-tools--unit-) - * [Test Tools (Integration)](#test-tools--integration-) - * [Validate Spec](#validate-spec) +- [Developer Guide](#developer-guide) + - [Getting Started](#getting-started) + - [Specification](#specification) + - [File Structure](#file-structure) + - [Grouping Operations](#grouping-operations) + - [Grouping Schemas](#grouping-schemas) + - [Superseded Operations](#superseded-operations) + - [Global Parameters](#global-parameters) + - [OpenAPI Extensions](#openapi-extensions) + - [Writing Spec Tests](#writing-spec-tests) + - [Tools](#tools) + - [Setup](#setup) + - [Spec Merger](#spec-merger) + - [Spec Linter](#spec-linter) + - [Spec Tester](#spec-tester) + - [Spec Style](#spec-style) + - [Dump Cluster Spec](#dump-cluster-spec) + - [Coverage](#coverage) + - [Tools Testing](#tools-testing) + - [Tools Linting](#tools-linting) + - [Workflows](#workflows) + - [Analyze PR Changes](#analyze-pr-changes) + - [Build](#build) + - [Deploy GitHub Pages](#deploy-github-pages) + - [Comment on PR](#comment-on-pr) + - [Test Tools (Unit)](#test-tools-unit) + - [Test Tools (Integration)](#test-tools-integration) + - [Validate Spec](#validate-spec) # Developer Guide @@ -173,13 +166,13 @@ npm run merge -- --help The merger tool merges the multi-file OpenSearch spec into a single file for programmatic use. -#### Arguments +**Arguments** - `--source `: The path to the root folder of the multi-file spec, defaults to `/spec`. - `--output `: The path to write the final merged spec to, defaults to `/build/opensearch-openapi.yaml`. - `--opensearch-version`: An optional target version of OpenSearch, checking values of `x-version-added` and `x-version-removed`. -#### Example +**Example** We can take advantage of the default values and simply merge the specification via: ```bash @@ -200,11 +193,11 @@ npm run lint:spec -- --help The linter tool validates the OpenSearch multi-file spec, and will print out all the errors and warnings in it. -#### Arguments +**Arguments** - `--source `: The path to the root folder of the multi-file spec, defaults to `/spec`. -#### Example +**Example** We can take advantage of the default values and simply lint the specification via: ```bash @@ -219,6 +212,20 @@ npm run test:spec -- --help The spec test framework validates the OpenSearch spec against a running OpenSearch cluster. As you modify the spec, you should add or update the spec test stories in the [./tests](tests) directory. For information on this topic, see [TESTING_GUIDE.md](TESTING_GUIDE.md). +### Spec Style + +This repo runs [Vale](https://github.com/errata-ai/vale) on the text contents of the spec, such as descriptions. + +The [Style prepare tool](tools/src/prepare-for-vale/) clears YAML files from all markup and leaves text in-place in the [style workflow](.github/workflows/style.yml), allowing for comments to appear in pull requests on GitHub. + +```bash +npm run style:prepare -- --help +``` + +**Arguments** + +- `--source `: The path to the root folder of the multi-file spec, defaults to `/spec`. + ### [Dump Cluster Spec](tools/src/dump-cluster-spec) ```bash @@ -227,7 +234,7 @@ npm run dump-cluster-spec -- --help The dump-cluster-spec tool connects to an OpenSearch cluster which has the [opensearch-api plugin](https://github.com/dblock/opensearch-api) installed and dumps the skeleton OpenAPI specification it provides to a file. -#### Arguments +**Arguments** - `--opensearch-url `: The URL at which the cluster is accessible, defaults to `https://localhost:9200`. - `--opensearch-insecure`: Disable SSL/TLS certificate verification, defaults to performing verification. @@ -235,7 +242,7 @@ The dump-cluster-spec tool connects to an OpenSearch cluster which has the [open - `--opensearch-password `: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable. - `--output `: The path to write the dumped spec to, defaults to `/build/opensearch-openapi-CLUSTER.yaml`. -#### Example +**Example** You can use this repo's [docker image which includes the opensearch-api plugin](coverage/Dockerfile) to spin up a local development cluster with a self-signed certificate (e.g. `https://localhost:9200`) and security enabled, to then dump the skeleton specification: ```bash @@ -264,13 +271,13 @@ npm run coverage:spec -- --help The coverage tool determines which APIs from the OpenSearch cluster's reference skeleton specification (dumped by the [dump-cluster-spec tool](#dump-cluster-spec)) are covered by this specification (as built by the [merger tool](#merger)). -#### Arguments +**Arguments** - `--cluster `: The path to the cluster's reference skeleton specification, as dumped by [dump-cluster-spec](#dump-cluster-spec), defaults to `/build/opensearch-openapi-CLUSTER.yaml`. - `--specification `: The path to the merged specification, as built by [merger](#merger), defaults to `/build/opensearch-openapi.yaml`. - `--output `: The path to write the coverage data to, defaults to `/build/coverage.json`. -#### Example +**Example** Assuming you've already followed the previous examples to build the merged specification with the [merger](#example) and dump the cluster's specification with [dump-cluster-spec](#example-2), you can then calculate the API coverage: ```bash diff --git a/package-lock.json b/package-lock.json index a6dae5581..7e903482a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", "eslint-plugin-yml": "^1.14.0", + "fast-glob": "^3.3.2", "globals": "^15.0.0", "json-diff-ts": "^4.0.1", "json-schema-to-typescript": "^14.0.4", diff --git a/package.json b/package.json index 5244756f3..a0715e2a2 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "lint": "eslint . --report-unused-disable-directives", "lint--fix": "eslint . --fix --report-unused-disable-directives", "merge": "ts-node tools/src/merger/merge.ts", + "style:prepare": "ts-node tools/src/prepare-for-vale/prepare-for-vale.ts", "test": "npm run test:unit && npm run test:integ", "jest": "jest", "test:unit": "jest --testMatch='**/*.test.ts' --testPathIgnorePatterns=/integ/", @@ -34,12 +35,12 @@ "@types/titlecase": "^1.1.2", "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^6.21.0", - "axios-mock-adapter": "^2.0.0", "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", "aws4-axios": "^3.3.7", "axios": "^1.7.5", + "axios-mock-adapter": "^2.0.0", "cbor": "^9.0.2", "commander": "^12.0.0", "eslint": "^8.57.0", @@ -51,6 +52,7 @@ "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", "eslint-plugin-yml": "^1.14.0", + "fast-glob": "^3.3.2", "globals": "^15.0.0", "json-diff-ts": "^4.0.1", "json-schema-to-typescript": "^14.0.4", diff --git a/tools/src/prepare-for-vale/KeepDescriptions.ts b/tools/src/prepare-for-vale/KeepDescriptions.ts new file mode 100644 index 000000000..6d6da3ae2 --- /dev/null +++ b/tools/src/prepare-for-vale/KeepDescriptions.ts @@ -0,0 +1,55 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import fs from 'fs' +import fg from 'fast-glob' +import { Logger } from '../Logger' + +/** + * Keeps only description: field values. + */ +export default class KeepDescriptions { + root_folder: string + logger: Logger + + constructor (root_folder: string, logger: Logger = new Logger()) { + this.logger = logger + this.root_folder = fs.realpathSync(root_folder) + } + + process(): void { + this.root_folder + const files = fg.globSync([`${this.root_folder}/**/*.yaml`]) + files.forEach((path) => { + this.logger.log(path) + this.process_file(path) + }) + } + + process_file(filename: string): void { + const contents = fs.readFileSync(filename, 'utf-8') + var writer = fs.openSync(filename, 'w+') + + var inside_description = false + contents.split(/\r?\n/).forEach((line) => { + if (line.match(/^[\s]+(description: \|)/)) { + inside_description = true + } else if (line.match(/^[\s]+(description:)[\s]+/)) { + fs.writeSync(writer, line.replace("description:", " ")) + } else if (inside_description && line.match(/^[\s]*[\w]*:/)) { + inside_description = false + } else if (inside_description) { + fs.writeSync(writer, line) + } + if (line.length > 0) { + fs.writeSync(writer, "\n") + } + }) + } +} diff --git a/tools/src/prepare-for-vale/prepare-for-vale.ts b/tools/src/prepare-for-vale/prepare-for-vale.ts new file mode 100644 index 000000000..9c6c3ceb0 --- /dev/null +++ b/tools/src/prepare-for-vale/prepare-for-vale.ts @@ -0,0 +1,27 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { Command, Option } from '@commander-js/extra-typings' +import { Logger, LogLevel } from '../Logger' +import { resolve } from 'path' +import KeepDescriptions from './KeepDescriptions' + +const command = new Command() + .description('Convert YAML files to text.') + .addOption(new Option('-s, --source ', 'path to the root folder of the multi-file spec').default(resolve(__dirname, '../../../spec'))) + .addOption(new Option('--verbose', 'show merge details').default(false)) + .allowExcessArguments(false) + .parse() + +const opts = command.opts() +const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) +const keep_descriptions = new KeepDescriptions(opts.source, logger) +logger.log(`Preparing ${opts.source} ...`) +keep_descriptions.process() +logger.log('Done.') diff --git a/tools/tests/prepare-for-vale/KeepDescriptions.test.ts b/tools/tests/prepare-for-vale/KeepDescriptions.test.ts new file mode 100644 index 000000000..0c595299b --- /dev/null +++ b/tools/tests/prepare-for-vale/KeepDescriptions.test.ts @@ -0,0 +1,43 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import KeepDescriptions from 'prepare-for-vale/KeepDescriptions' +import fs from 'fs' +import fg from 'fast-glob' +import tmp from 'tmp' + +describe('KeepDescriptions', () => { + var temp: tmp.DirResult + var fixture_path: string = './tools/tests/prepare-for-vale/fixtures' + var fixtures = fg.globSync(`${fixture_path}/**/*.yaml`) + + describe('defaults', () => { + beforeAll(() => { + temp = tmp.dirSync() + fs.cpSync(fixture_path, temp.name, { recursive: true }) + new KeepDescriptions(temp.name).process() + }) + + afterAll(() => { + temp.removeCallback() + }) + + describe('converts files to text in-place', () => { + fixtures.forEach((filename) => { + test(filename, () => { + const processed_yaml = filename.replace(fixture_path, temp.name) + const filename_txt = processed_yaml.replace(".yaml", ".txt") + expect(fs.readFileSync(processed_yaml, 'utf8')).toEqual(fs.readFileSync(filename_txt, 'utf8')) + fs.rmSync(processed_yaml) + fs.rmSync(filename_txt) + }) + }) + }) + }) +}) \ No newline at end of file diff --git a/tools/tests/prepare-for-vale/fixtures/spec.txt b/tools/tests/prepare-for-vale/fixtures/spec.txt new file mode 100644 index 000000000..a54608e03 --- /dev/null +++ b/tools/tests/prepare-for-vale/fixtures/spec.txt @@ -0,0 +1,26 @@ + + + + A description that is preserved. + + + + + + + + For a successful response, this value is always true. On failure, an exception is returned instead. + + + + The item level REST category class codes during indexing. + + + + Line one + Line two + + + + Line one + Line with a description: that describes itself. diff --git a/tools/tests/prepare-for-vale/fixtures/spec.yaml b/tools/tests/prepare-for-vale/fixtures/spec.yaml new file mode 100644 index 000000000..8801c55e2 --- /dev/null +++ b/tools/tests/prepare-for-vale/fixtures/spec.yaml @@ -0,0 +1,26 @@ +openapi: 3.1.0 +info: + title: A title that is preserved. + description: A description that is preserved. + version: 1.0.0 +components: + schemas: + ObjectWithDescription: + type: object + properties: + acknowledged: + description: For a successful response, this value is always true. On failure, an exception is returned instead. + type: boolean + ObjectWithMultilineDescriptionOneLine: + description: |- + The item level REST category class codes during indexing. + type: object + ObjectWithMultilineDescriptionTwoLines: + description: |- + Line one + Line two + ObjectWithAColonInDescription: + type: object + description: |- + Line one + Line with a description: that describes itself. diff --git a/tools/tests/prepare-for-vale/prepare-for-vale.test.ts b/tools/tests/prepare-for-vale/prepare-for-vale.test.ts new file mode 100644 index 000000000..c46fc7340 --- /dev/null +++ b/tools/tests/prepare-for-vale/prepare-for-vale.test.ts @@ -0,0 +1,22 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { spawnSync } from 'child_process' + +const spec = (args: string[]): any => { + const start = spawnSync('ts-node', ['tools/src/prepare-for-vale/prepare-for-vale.ts'].concat(args)) + return { + stdout: start.stdout?.toString(), + stderr: start.stderr?.toString() + } +} + +test('--help', () => { + expect(spec(['--help']).stdout).toContain('Usage: prepare-for-vale [options]') +}) From 552e5e4366c2429a58ec91bf10c466d8dffde9c4 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 16:58:26 -0400 Subject: [PATCH 61/73] Adds tests for /_remote/info and _script_context. (#637) * tests/default/_core/ refresh, remote_info and script_context Signed-off-by: Bruno Lloret Signed-off-by: dblock Co-authored-by: Bruno Lloret --- tests/default/_core/refresh.yaml | 10 +++++++++ tests/default/_core/remote/info.yaml | 27 +++++++++++++++++++++++++ tests/default/_core/script_context.yaml | 10 +++++++++ tests/default/indices/refresh.yaml | 5 ----- 4 files changed, 47 insertions(+), 5 deletions(-) create mode 100644 tests/default/_core/refresh.yaml create mode 100644 tests/default/_core/remote/info.yaml create mode 100644 tests/default/_core/script_context.yaml diff --git a/tests/default/_core/refresh.yaml b/tests/default/_core/refresh.yaml new file mode 100644 index 000000000..5a4a42f74 --- /dev/null +++ b/tests/default/_core/refresh.yaml @@ -0,0 +1,10 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test _refresh. + +chapters: + - synopsis: Refresh all indices in the cluster. + path: /_refresh + method: GET + response: + status: 200 diff --git a/tests/default/_core/remote/info.yaml b/tests/default/_core/remote/info.yaml new file mode 100644 index 000000000..708439460 --- /dev/null +++ b/tests/default/_core/remote/info.yaml @@ -0,0 +1,27 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test _remote/info. + +prologues: + - method: PUT + path: /_cluster/settings + request: + payload: + transient: + cluster.remote: + cluster2: + seeds: + - localhost:4242 +chapters: + - synopsis: Retrieve information about remote clusters. + path: /_remote/info + method: GET + response: + status: 200 +epilogues: + - method: PUT + path: /_cluster/settings + request: + payload: + transient: + cluster.remote.cluster2.seeds: [] diff --git a/tests/default/_core/script_context.yaml b/tests/default/_core/script_context.yaml new file mode 100644 index 000000000..bcb842edb --- /dev/null +++ b/tests/default/_core/script_context.yaml @@ -0,0 +1,10 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test the _script_context endpoint to retrieve available script contexts. + +chapters: + - synopsis: Retrieve available script contexts. + path: /_script_context + method: GET + response: + status: 200 \ No newline at end of file diff --git a/tests/default/indices/refresh.yaml b/tests/default/indices/refresh.yaml index 2303ed287..eb8f69057 100644 --- a/tests/default/indices/refresh.yaml +++ b/tests/default/indices/refresh.yaml @@ -14,11 +14,6 @@ prologues: year: 1991 status: [201] chapters: - - synopsis: Refresh. - path: /_refresh - method: POST - response: - status: 200 - synopsis: Refresh an index. path: /{index}/_refresh method: POST From 1db184063a463c5180a2cc824b1efc1aeebfd5eb Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 24 Oct 2024 17:03:50 -0400 Subject: [PATCH 62/73] Do not run Vale on push. (#645) Signed-off-by: dblock --- .github/workflows/style.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 7bce3cd1a..8ebc71202 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -1,6 +1,6 @@ name: Style -on: [pull_request, push] +on: [pull_request] jobs: check: From 86e4789f558a4d8237219ff9ce558b17f5e00242 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 10:30:29 -0400 Subject: [PATCH 63/73] Preparing for next developer iteration 0.1.1. (#647) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dblock <542335+dblock@users.noreply.github.com> --- CHANGELOG.md | 7 +++++-- spec/_info.yaml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6448aa8ec..323fa2686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] +## [0.1.0] - 2024-10-25 + ### Added - Added CHANGELOG ([#309](https://github.com/opensearch-project/opensearch-api-specification/pull/309)) @@ -140,7 +142,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Removed `shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) - Removed unsupported DataStream Lifecycle types ([#600](https://github.com/opensearch-project/opensearch-api-specification/pull/600)) - Removed unsupported runtime field types ([#634](https://github.com/opensearch-project/opensearch-api-specification/pull/634)) - + ### Fixed - Fixed GitHub pages ([#215](https://github.com/opensearch-project/opensearch-api-specification/pull/215)) @@ -187,4 +189,5 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Security -[Unreleased]: https://github.com/opensearch-project/opensearch-api-specification/commits/main +[unreleased]: https://github.com/opensearch-project/opensearch-api-specification/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/opensearch-project/opensearch-api-specification/compare/1db184063a463c5180a2cc824b1efc1aeebfd5eb...v0.1.0 diff --git a/spec/_info.yaml b/spec/_info.yaml index 58818da48..f9d169602 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -1,5 +1,5 @@ $schema: ./json_schemas/_info.schema.yaml title: OpenSearch API Specification -version: 0.1.0 +version: 0.1.1 x-api-version: 2.16.0 From 0605aec82fca2673ded1a138da113cfc4abdfc45 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:00:02 -0500 Subject: [PATCH 64/73] [Docs]: Update rest of core parameter descriptions (#623) * [Docs]: Update rest of core parameter descriptions Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Update spec/schemas/_core.reindex_rethrottle.yaml Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Apply suggestions from code review Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Update spec/schemas/_core.search.yaml Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower --- spec/schemas/_core.reindex.yaml | 2 +- spec/schemas/_core.reindex_rethrottle.yaml | 8 ++-- .../_core.scripts_painless_execute.yaml | 2 +- spec/schemas/_core.search.yaml | 46 +++++++++---------- spec/schemas/_core.termvectors.yaml | 10 ++-- 5 files changed, 34 insertions(+), 34 deletions(-) diff --git a/spec/schemas/_core.reindex.yaml b/spec/schemas/_core.reindex.yaml index fc4a1ae2e..eaa2c1579 100644 --- a/spec/schemas/_core.reindex.yaml +++ b/spec/schemas/_core.reindex.yaml @@ -34,7 +34,7 @@ components: size: description: |- The number of documents to index per batch. - Use when indexing from remote to ensure that the batches fit within the on-heap buffer, which defaults to a maximum size of 100 MB. + Use the `size` setting when indexing from a remote cluster. This ensures that batches fit in the on-heap buffer. The buffer defaults to a maximum size of `100MB`. type: number slice: $ref: '_common.yaml#/components/schemas/SlicedScroll' diff --git a/spec/schemas/_core.reindex_rethrottle.yaml b/spec/schemas/_core.reindex_rethrottle.yaml index 1fadb4aa0..6275fef00 100644 --- a/spec/schemas/_core.reindex_rethrottle.yaml +++ b/spec/schemas/_core.reindex_rethrottle.yaml @@ -55,7 +55,7 @@ components: type: object properties: batches: - description: The number of scroll responses pulled back by the reindex. + description: The number of scroll responses shown by the reindex. type: number created: description: The number of documents that were successfully created. @@ -64,10 +64,10 @@ components: description: The number of documents that were successfully deleted. type: number noops: - description: The number of documents that were ignored because the script used for the reindex returned a `noop` value for `ctx.op`. + description: The number of documents that were ignored because the script used for the reindex operation returned a `noop` value for `ctx.op`. type: number requests_per_second: - description: The number of requests per second effectively executed during the reindex. + description: The number of successful requests per second during the reindex operation. type: number retries: $ref: '_common.yaml#/components/schemas/Retries' @@ -83,7 +83,7 @@ components: description: The number of documents that were successfully processed. type: number updated: - description: The number of documents that were successfully updated, for example, a document with same ID already existed prior to reindex updating it. + description: The number of documents that were successfully updated. type: number version_conflicts: description: The number of version conflicts that reindex hits. diff --git a/spec/schemas/_core.scripts_painless_execute.yaml b/spec/schemas/_core.scripts_painless_execute.yaml index 5997842ff..8aca44805 100644 --- a/spec/schemas/_core.scripts_painless_execute.yaml +++ b/spec/schemas/_core.scripts_painless_execute.yaml @@ -10,7 +10,7 @@ components: type: object properties: document: - description: Document that's temporarily indexed in-memory and accessible from the script. + description: A document temporarily indexed in-memory and accessible from the Painless script. type: object index: $ref: '_common.yaml#/components/schemas/IndexName' diff --git a/spec/schemas/_core.search.yaml b/spec/schemas/_core.search.yaml index a9d1f9077..ab84bebc0 100644 --- a/spec/schemas/_core.search.yaml +++ b/spec/schemas/_core.search.yaml @@ -10,7 +10,7 @@ components: type: object properties: total: - description: Total hit count information, present only if `track_total_hits` wasn't `false` in the search request. + description: The total number of hits, present only if `track_total_hits` is not set to `false` in the search request. oneOf: - $ref: '#/components/schemas/TotalHits' - type: integer @@ -639,10 +639,10 @@ components: - text TrackHits: description: |- - Number of hits matching the query to count accurately. If true, the exact - number of hits is returned at the cost of some performance. If false, the + The number of hits matching the query. When `true`, the exact + number of hits is returned at the cost of some performance. When `false`, the response does not include the total number of hits matching the query. - Defaults to 10,000 hits. + Default is `10,000` hits. oneOf: - type: boolean - type: integer @@ -650,7 +650,7 @@ components: SourceConfigParam: description: |- Defines how to fetch a source. Fetching can be disabled entirely, or the source can be filtered. - Used as a query parameter along with the `_source_includes` and `_source_excludes` parameters. + Use this setting with the `_source_includes` and `_source_excludes` parameters. oneOf: - type: boolean - $ref: '_common.yaml#/components/schemas/Fields' @@ -664,7 +664,7 @@ components: type: integer format: int32 from: - description: Inner hit starting document offset. + description: The inner hit that initiates document offset. type: integer format: int32 collapse: @@ -703,14 +703,14 @@ components: field: $ref: '_common.yaml#/components/schemas/Field' inner_hits: - description: The number of inner hits and their sort order + description: The number of inner hits and their sort order. oneOf: - $ref: '#/components/schemas/InnerHits' - type: array items: $ref: '#/components/schemas/InnerHits' max_concurrent_group_searches: - description: The number of concurrent requests allowed to retrieve the inner_hits per group + description: The number of concurrent requests that are allowed to be retrieved by the `inner_hits` parameter per group. type: integer required: - field @@ -761,7 +761,7 @@ components: boundary_scanner_locale: description: |- Controls which locale is used to search for sentence and word boundaries. - This parameter takes a form of a language tag, for example: `"en-US"`, `"fr-FR"`, `"ja-JP"`. + This parameter takes the form of a language tag, for example, `"en-US"`, `"fr-FR"`, or `"ja-JP"`. type: string force_source: deprecated: true @@ -782,8 +782,8 @@ components: max_analyzed_offset: description: |- If set to a non-negative value, highlighting stops at this defined maximum limit. - The rest of the text is not processed, thus not highlighted and no error is returned - The `max_analyzed_offset` query setting does not override the `index.highlight.max_analyzed_offset` setting, which prevails when it's set to lower value than the query setting. + The rest of the text is not processed or highlighted, and no error is returned. + The `max_analyzed_offset` query setting does not override the `index.highlight.max_analyzed_offset` setting, which takes precedence when it is set to a lower value than the query setting. type: integer format: int32 no_match_size: @@ -793,10 +793,10 @@ components: number_of_fragments: description: |- The maximum number of fragments to return. - If the number of fragments is set to `0`, no fragments are returned. - Instead, the entire field contents are highlighted and returned. - This can be handy when you need to highlight short texts such as a title or address, but fragmentation is not required. - If `number_of_fragments` is `0`, `fragment_size` is ignored. + When the number of fragments is set to `0`, no fragments are returned. + Instead, the entirety of a field's contents are highlighted and returned. + This is useful when you need to highlight short texts, such as a title or address, in which fragmentation is not required. + If `number_of_fragments` is set to `0`, the `fragment_size` is ignored. type: integer format: int32 options: @@ -808,21 +808,21 @@ components: phrase_limit: description: |- Controls the number of matching phrases in a document that are considered. - Prevents the `fvh` highlighter from analyzing too many phrases and consuming too much memory. - When using `matched_fields`, `phrase_limit` phrases per matched field are considered. Raising the limit increases query time and consumes more memory. - Only supported by the `fvh` highlighter. + This prevents the `fvh` highlighter from analyzing too many phrases and consuming too much memory. + When using `matched_fields`, phrase-limited phrases per matched field are considered. Raising the limit increases the query time and consumes more memory. + This setting is only supported by the `fvh` highlighter. type: integer format: int32 post_tags: description: |- - Use in conjunction with `pre_tags` to define the HTML tags to use for the highlighted text. + When used in conjunction with `pre_tags`, defines the HTML tags to use for the highlighted text. By default, highlighted text is wrapped in `` and `` tags. type: array items: type: string pre_tags: description: |- - Use in conjunction with `post_tags` to define the HTML tags to use for the highlighted text. + When used in conjunction with `post_tags`, defines the HTML tags to use for the highlighted text. By default, highlighted text is wrapped in `` and `` tags. type: array items: @@ -899,11 +899,11 @@ components: rescore_query: $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' query_weight: - description: Relative importance of the original query versus the rescore query. + description: The relative importance of the original query as compared to the rescore query. type: number format: float rescore_query_weight: - description: Relative importance of the rescore query versus the original query. + description: The relative importance of the rescore query as compared to the original query. type: number format: float score_mode: @@ -922,7 +922,7 @@ components: type: object properties: text: - description: Global suggest text, to avoid repetition when the same text is used in several suggesters + description: The global suggest text, which avoids repetition when the same text is used in several suggesters. type: string PointInTimeReference: type: object diff --git a/spec/schemas/_core.termvectors.yaml b/spec/schemas/_core.termvectors.yaml index 7cca65878..d7a309b9a 100644 --- a/spec/schemas/_core.termvectors.yaml +++ b/spec/schemas/_core.termvectors.yaml @@ -11,21 +11,21 @@ components: properties: max_doc_freq: description: |- - Ignore words which occur in more than this many docs. - Defaults to unbounded. + Ignores words that appear in more than the specified number of documents. + Default is `unbounded`. type: number max_num_terms: - description: Maximum number of terms that must be returned per field. + description: The maximum number of terms that should be returned per field. type: number max_term_freq: description: |- Ignore words with more than this frequency in the source doc. - Defaults to unbounded. + Default is `unbounded`. type: number max_word_length: description: |- The maximum word length above which words will be ignored. - Defaults to unbounded. + Default is `unbounded`. type: number min_doc_freq: description: Ignore terms which do not occur in at least this many docs. From 931a61ae023b453e3611f37a802ef27b9c4877a1 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Fri, 25 Oct 2024 13:34:08 -0500 Subject: [PATCH 65/73] Reopen mtern descriptions PR for editorial. (#648) * Reopen mtern descriptions PR for editorial. Signed-off-by: Archer * Apply suggestions from code review Co-authored-by: Nathan Bower Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> * Remove spaces Signed-off-by: Archer --------- Signed-off-by: Archer Signed-off-by: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Co-authored-by: Nathan Bower --- spec/schemas/_core.mtermvectors.yaml | 2 +- spec/schemas/_core.rank_eval.yaml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/spec/schemas/_core.mtermvectors.yaml b/spec/schemas/_core.mtermvectors.yaml index ced45db7e..e130a31f5 100644 --- a/spec/schemas/_core.mtermvectors.yaml +++ b/spec/schemas/_core.mtermvectors.yaml @@ -14,7 +14,7 @@ components: _index: $ref: '_common.yaml#/components/schemas/IndexName' doc: - description: An artificial document for which you want to retrieve term vectors. + description: An artificial document for which you want to retrieve term vectors. type: object fields: $ref: '_common.yaml#/components/schemas/Fields' diff --git a/spec/schemas/_core.rank_eval.yaml b/spec/schemas/_core.rank_eval.yaml index d5e1d6eb0..404d09c3f 100644 --- a/spec/schemas/_core.rank_eval.yaml +++ b/spec/schemas/_core.rank_eval.yaml @@ -21,7 +21,7 @@ components: template_id: $ref: '_common.yaml#/components/schemas/Id' params: - description: The search template parameters. + description: The search template parameters. type: object additionalProperties: type: object @@ -70,7 +70,7 @@ components: - type: object properties: ignore_unlabeled: - description: Controls how unlabeled documents in the search results are counted. When `true`, unlabeled documents are ignored and do not count as relevant or irrelevant. When `false`, unlabeled documents are treated as irrelevant. + description: Controls how unlabeled documents in the search results are counted. When `true`, unlabeled documents are ignored and are not treated as relevant or irrelevant. When `false`, unlabeled documents are treated as irrelevant. type: boolean RankEvalMetricRatingThreshold: allOf: @@ -84,7 +84,7 @@ components: type: object properties: k: - description: Sets the maximum number of documents retrieved per query. This value replaces the usual `size` parameter in the query. + description: Sets the maximum number of documents retrieved per query. This value replaces the `size` parameter in the query. type: number RankEvalMetricRecall: allOf: @@ -100,7 +100,7 @@ components: - type: object properties: normalize: - description: When `true`, this metric calculates the Normalized DCG (https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG). + description: When `true`, calculates the [normalized discounted cumulative gain (nDCG)](https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG). type: boolean RankEvalMetricExpectedReciprocalRank: allOf: @@ -108,7 +108,7 @@ components: - type: object properties: maximum_relevance: - description: The highest relevance grade used in the user-supplied relevance judgments. + description: The highest relevance grade used in user-supplied relevance judgments. type: number required: - maximum_relevance @@ -119,17 +119,17 @@ components: description: The `metric_score`, found in the `metric_details` section, shows the contribution of this query to the global quality metric score. type: number unrated_docs: - description: The `unrated_docs` section contains an `_index` and `_id` entry for each document that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents. + description: The `unrated_docs` section contains an `_index` and `_id` entry for each document that didn't have a `ratings` value. This can be used to ask the user to supply ratings for these documents. type: array items: $ref: '#/components/schemas/UnratedDocument' hits: - description: The `hits` section shows a grouping of the search results with their supplied ratings + description: The `hits` section provides a grouping of the search results with their supplied ratings. type: array items: $ref: '#/components/schemas/RankEvalHitItem' metric_details: - description: The `metric_details` section gives additional information about the calculated quality metric, in other words, how many of the retrieved documents were relevant. The content varies for each metric but allows for better interpretation of the results. + description: The `metric_details` section provides additional information about the calculated quality metric indicating the number of relevant retrieved documents. The content varies for each metric but allows for better interpretation of the results. type: object additionalProperties: type: object From 5e3349a11c297dcf9a690853c896c2099f916cbd Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Wed, 30 Oct 2024 14:23:37 -0700 Subject: [PATCH 66/73] add workload management querygroup api specs (#649) * add wlm querygroup api specs Signed-off-by: Ruirui Zhang * fix issues Signed-off-by: Ruirui Zhang * address comments Signed-off-by: Ruirui Zhang * change folder name Signed-off-by: Ruirui Zhang --- .github/workflows/test-spec.yml | 2 + CHANGELOG.md | 1 + spec/namespaces/wlm.yaml | 127 ++++++++++++++++++ spec/schemas/wlm._common.yaml | 100 ++++++++++++++ tests/plugins/workload-management/Dockerfile | 6 + .../workload-management/docker-compose.yml | 17 +++ .../workload-management/query_group.yaml | 67 +++++++++ 7 files changed, 320 insertions(+) create mode 100644 spec/namespaces/wlm.yaml create mode 100644 spec/schemas/wlm._common.yaml create mode 100644 tests/plugins/workload-management/Dockerfile create mode 100644 tests/plugins/workload-management/docker-compose.yml create mode 100644 tests/plugins/workload-management/query_group.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 23f0fffc6..3729b66cb 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -42,6 +42,8 @@ jobs: tests: plugins/notifications - version: 2.17.0 tests: plugins/query_insights + - version: 2.17.0 + tests: plugins/workload-management - version: 2.18.0 hub: opensearchstaging ref: '@sha256:57a1cd1142d68c203e2e4aa0666d9a691e1e409a5d14aa4a8f5044036f05cf06' diff --git a/CHANGELOG.md b/CHANGELOG.md index 323fa2686..52a61fab2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) - Added API spec for query insights plugin ([#625](https://github.com/opensearch-project/opensearch-api-specification/pull/625)) +- Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) - Added API spec for `adjust_pure_negative` for bool queries ([#641](https://github.com/opensearch-project/opensearch-api-specification/pull/641)) - Added a spec style checker [#620](https://github.com/opensearch-project/opensearch-api-specification/pull/620). - Added `remote_store` to node `Stats` ([#643](https://github.com/opensearch-project/opensearch-api-specification/pull/643)) diff --git a/spec/namespaces/wlm.yaml b/spec/namespaces/wlm.yaml new file mode 100644 index 000000000..083f8ed9e --- /dev/null +++ b/spec/namespaces/wlm.yaml @@ -0,0 +1,127 @@ +openapi: 3.1.0 +info: + title: OpenSearch Workload Management Query Group API + description: Query group lifecycle APIs for the workload management plugin. + version: 1.0.0 +paths: + /_wlm/query_group: + get: + operationId: wlm.get_query_group.0 + x-operation-group: wlm.get_query_group + x-version-added: '2.17' + description: Gets the specified QueryGroup or get all if no name is provided. + responses: + '200': + $ref: '#/components/responses/wlm.get_query_group@200' + put: + operationId: wlm.create_query_group.0 + x-operation-group: wlm.create_query_group + x-version-added: '2.17' + description: Creates the specified query group. + requestBody: + $ref: '#/components/requestBodies/wlm.create_query_group' + responses: + '200': + $ref: '#/components/responses/wlm.create_query_group@200' + /_wlm/query_group/{name}: + get: + operationId: wlm.get_query_group.1 + x-operation-group: wlm.get_query_group + x-version-added: '2.17' + description: Gets the specified QueryGroup or get all if no name is provided. + parameters: + - $ref: '#/components/parameters/wlm.get_query_group::path.name' + responses: + '200': + $ref: '#/components/responses/wlm.get_query_group@200' + put: + operationId: wlm.update_query_group.0 + x-operation-group: wlm.update_query_group + x-version-added: '2.17' + description: Updates the specified query group. + parameters: + - $ref: '#/components/parameters/wlm.update_query_group::path.name' + requestBody: + $ref: '#/components/requestBodies/wlm.update_query_group' + responses: + '200': + $ref: '#/components/responses/wlm.update_query_group@200' + delete: + operationId: wlm.delete_query_group.0 + x-operation-group: wlm.delete_query_group + x-version-added: '2.17' + description: Deletes the specified QueryGroup. + parameters: + - $ref: '#/components/parameters/wlm.delete_query_group::path.name' + responses: + '200': + $ref: '#/components/responses/wlm.delete_query_group@200' +components: + requestBodies: + wlm.create_query_group: + content: + application/json: + schema: + $ref: '../schemas/wlm._common.yaml#/components/schemas/QueryGroupCreate' + required: true + wlm.update_query_group: + content: + application/json: + schema: + $ref: '../schemas/wlm._common.yaml#/components/schemas/QueryGroupUpdate' + required: true + responses: + wlm.create_query_group@200: + description: '' + content: + application/json: + schema: + $ref: '../schemas/wlm._common.yaml#/components/schemas/QueryGroupResponse' + wlm.update_query_group@200: + description: '' + content: + application/json: + schema: + $ref: '../schemas/wlm._common.yaml#/components/schemas/QueryGroupResponse' + wlm.delete_query_group@200: + description: '' + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + wlm.get_query_group@200: + description: '' + content: + application/json: + schema: + type: object + properties: + query_groups: + type: array + items: + $ref: '../schemas/wlm._common.yaml#/components/schemas/QueryGroupResponse' + parameters: + wlm.update_query_group::path.name: + name: name + in: path + description: QueryGroup name. + schema: + type: string + required: true + wlm.delete_query_group::path.name: + name: name + in: path + description: QueryGroup name. + schema: + type: string + required: true + wlm.get_query_group::path.name: + name: name + in: path + description: QueryGroup name. + schema: + type: string + required: true diff --git a/spec/schemas/wlm._common.yaml b/spec/schemas/wlm._common.yaml new file mode 100644 index 000000000..ca2c1ae7a --- /dev/null +++ b/spec/schemas/wlm._common.yaml @@ -0,0 +1,100 @@ +openapi: 3.1.0 +info: + title: Schemas of wlm._common category + description: Schemas of wlm._common category + version: 1.0.0 +paths: {} +components: + schemas: + QueryGroupResponse: + type: object + properties: + _id: + type: string + name: + type: string + resiliency_mode: + type: string + enum: [enforced, monitor,soft] + updated_at: + type: integer + format: int64 + resource_limits: + anyOf: + - type: object + properties: + memory: + type: number + format: double + cpu: + type: number + format: double + required: + - memory + - type: object + properties: + memory: + type: number + format: double + cpu: + type: number + format: double + required: + - cpu + required: + - _id + - name + - resiliency_mode + - resource_limits + - updated_at + QueryGroupCreate: + type: object + properties: + name: + type: string + resiliency_mode: + type: string + enum: [enforced, monitor,soft] + resource_limits: + anyOf: + - type: object + properties: + memory: + type: number + format: double + cpu: + type: number + format: double + required: + - memory + - type: object + properties: + memory: + type: number + format: double + cpu: + type: number + format: double + required: + - cpu + required: + - name + - resiliency_mode + - resource_limits + QueryGroupUpdate: + type: object + properties: + resiliency_mode: + type: string + enum: [enforced, monitor,soft] + resource_limits: + $ref: '#/components/schemas/ResourceLimitsSchema' + ResourceLimitsSchema: + type: object + properties: + memory: + type: number + format: double + cpu: + type: number + format: double diff --git a/tests/plugins/workload-management/Dockerfile b/tests/plugins/workload-management/Dockerfile new file mode 100644 index 000000000..2d040ccb8 --- /dev/null +++ b/tests/plugins/workload-management/Dockerfile @@ -0,0 +1,6 @@ +ARG OPENSEARCH_DOCKER_HUB_PROJECT +ARG OPENSEARCH_VERSION +ARG OPENSEARCH_DOCKER_REF + +FROM ${OPENSEARCH_DOCKER_HUB_PROJECT}/opensearch:${OPENSEARCH_VERSION}${OPENSEARCH_DOCKER_REF} +RUN ./bin/opensearch-plugin install -b workload-management diff --git a/tests/plugins/workload-management/docker-compose.yml b/tests/plugins/workload-management/docker-compose.yml new file mode 100644 index 000000000..c334c477f --- /dev/null +++ b/tests/plugins/workload-management/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + opensearch-cluster: + build: + context: . + args: + - OPENSEARCH_DOCKER_HUB_PROJECT=${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject} + - OPENSEARCH_DOCKER_REF=${OPENSEARCH_DOCKER_REF} + - OPENSEARCH_VERSION=2.17.0 + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/plugins/workload-management/query_group.yaml b/tests/plugins/workload-management/query_group.yaml new file mode 100644 index 000000000..6f2b86367 --- /dev/null +++ b/tests/plugins/workload-management/query_group.yaml @@ -0,0 +1,67 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: This story tests all endpoints relevant to QueryGroups, from creation to deletion. + +prologues: + - path: /_wlm/query_group + version: '>= 2.17' + method: PUT + request: + payload: + name: analytics + resource_limits: + memory: 0.1 + resiliency_mode: monitor + status: [200] + - path: /_wlm/query_group + version: '>= 2.17' + method: PUT + request: + payload: + name: analytics2 + resource_limits: + memory: 0.15 + cpu: 0.05 + resiliency_mode: monitor + status: [200] + +chapters: + - synopsis: Update the QueryGroup named `analytics2`, with resiliency_mode to be enforced and memory to be 0.25 and cpu to be 0.25. + version: '>= 2.17' + path: /_wlm/query_group/{name} + method: PUT + parameters: + name: analytics2 + request: + payload: + resource_limits: + memory: 0.25 + cpu: 0.25 + resiliency_mode: enforced + response: + status: 200 + - synopsis: Get the QueryGroup named `analytics2`. + version: '>= 2.17' + path: /_wlm/query_group/{name} + method: GET + parameters: + name: analytics2 + response: + status: 200 + payload: + query_groups: + - name: analytics2 + resource_limits: + memory: 0.25 + cpu: 0.25 + resiliency_mode: enforced + +epilogues: + - path: /_wlm/query_group/analytics + version: '>= 2.17' + method: DELETE + status: [200] + - path: /_wlm/query_group/analytics2 + version: '>= 2.17' + method: DELETE + status: [200] From 58674f4c2aacd3984a7c20011292fc883bb633e4 Mon Sep 17 00:00:00 2001 From: Oleksandr Loyko Date: Wed, 30 Oct 2024 18:50:42 -0400 Subject: [PATCH 67/73] Adds tests for _msearch (#650) Signed-off-by: Alex Loyko --- tests/default/_core/msearch.yaml | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 tests/default/_core/msearch.yaml diff --git a/tests/default/_core/msearch.yaml b/tests/default/_core/msearch.yaml new file mode 100644 index 000000000..79d92c9b8 --- /dev/null +++ b/tests/default/_core/msearch.yaml @@ -0,0 +1,55 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test document multi-search. +epilogues: + - path: /books + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book1}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {create: {_index: movies, _id: movie1}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies, _id: movie2}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Run multiple search requests in single request. + path: /_msearch + method: GET + request: + content_type: application/x-ndjson + payload: + - {index: books} + - {query: {match_all: {}}} + - {index: movies} + - {query: {match_all: {}}} + response: + status: 200 + payload: + responses: + - hits: + total: + value: 1 + relation: eq + hits: + - _index: books + _id: book1 + - hits: + total: + value: 2 + relation: eq + hits: + - _index: movies + _id: movie1 + - _index: movies + _id: movie2 From 1eb43f6ad0ed6bfe408f0689e181e914ff874997 Mon Sep 17 00:00:00 2001 From: Ruirui Zhang Date: Thu, 31 Oct 2024 04:26:15 -0700 Subject: [PATCH 68/73] modify changelog (#651) Signed-off-by: Ruirui Zhang --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52a61fab2..598dd71a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] +### Added +- Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) + ## [0.1.0] - 2024-10-25 ### Added @@ -106,7 +109,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `config_id` and `config_id_list` to `/_plugins/_notifications/configs` query parameters ([#594](https://github.com/opensearch-project/opensearch-api-specification/pull/594)) - Added a release workflow triggered on a tag ([#635](https://github.com/opensearch-project/opensearch-api-specification/pull/635)) - Added API spec for query insights plugin ([#625](https://github.com/opensearch-project/opensearch-api-specification/pull/625)) -- Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) - Added API spec for `adjust_pure_negative` for bool queries ([#641](https://github.com/opensearch-project/opensearch-api-specification/pull/641)) - Added a spec style checker [#620](https://github.com/opensearch-project/opensearch-api-specification/pull/620). - Added `remote_store` to node `Stats` ([#643](https://github.com/opensearch-project/opensearch-api-specification/pull/643)) From 07e329e8d01fd0576de6a0a3c35412fd5a9163db Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Fri, 1 Nov 2024 10:04:43 +1300 Subject: [PATCH 69/73] Remove SourceField mode (#652) * Remove SourceField mode Signed-off-by: Thomas Farr * Add PR number Signed-off-by: Thomas Farr * Bump version Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 3 +++ spec/_info.yaml | 2 +- spec/schemas/_common.mapping.yaml | 8 -------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 598dd71a3..5bc71eaeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Added - Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) +### Removed +- Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) + ## [0.1.0] - 2024-10-25 ### Added diff --git a/spec/_info.yaml b/spec/_info.yaml index f9d169602..81d079772 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -1,5 +1,5 @@ $schema: ./json_schemas/_info.schema.yaml title: OpenSearch API Specification -version: 0.1.1 +version: 0.2.0 x-api-version: 2.16.0 diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index 2ec2dfbda..6523b2143 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -1202,14 +1202,6 @@ components: type: array items: type: string - mode: - $ref: '#/components/schemas/SourceFieldMode' - SourceFieldMode: - type: string - enum: - - disabled - - stored - - synthetic DataStreamTimestamp: type: object properties: From 316757763cec208e9c3b1d23de3bacc6e28e2816 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 6 Nov 2024 08:37:53 -0500 Subject: [PATCH 70/73] Document the high level status and goals of this repo. (#655) Signed-off-by: dblock --- README.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7b05be832..ca512a1e3 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ - [OpenSearch API Specification](#opensearch-api-specification) - [Welcome!](#welcome) + - [OpenSearch API Source of Truth](#opensearch-api-source-of-truth) + - [Working in this Repo](#working-in-this-repo) - [Project Resources](#project-resources) - [Code of Conduct](#code-of-conduct) - [Security](#security) @@ -18,10 +20,15 @@ ## Welcome! -The `opensearch-api-specification` is an open source, community-driven collection of API model specifications for [OpenSearch](https://github.com/opensearch-project/OpenSearch) APIs. The API models are written in OpenAPI format and are used to generate client libraries and documentation. You can find the latest version of the API specification [here](https://github.com/opensearch-project/opensearch-api-specification/releases/download/main-latest/opensearch-openapi.yaml). +The `opensearch-api-specification` is an open source, community-driven collection of API model specifications for [OpenSearch](https://github.com/opensearch-project/OpenSearch) APIs. The API models are written in OpenAPI format and are used to generate client libraries and documentation. You can find the latest release of the API specification [here](https://github.com/opensearch-project/opensearch-api-specification/releases). -To contribute to this project or to track the developments head over to [Projects](https://github.com/opensearch-project/opensearch-api-specification/projects) board. Follow the [Developer guide](DEVELOPER_GUIDE.md) and [Contributing guidelines](CONTRIBUTING.md) for instructions -on building and contributing to opensearch-api-specification. +### OpenSearch API Source of Truth + +This repo aims to be the complete source of truth for OpenSearch and OpenSearch Dashboards REST APIs, including plugins. To be the source of truth, this repo [accurately represents APIs in YAML](spec/), and [publishes](https://github.com/opensearch-project/opensearch-api-specification/releases) a single-file OpenAPI 3.1.0 spec. The latter is used to [generate OpenSearch language clients](https://github.com/opensearch-project/opensearch-clients/issues/19), [generate mechanical parts of the OpenSearch documentation](https://github.com/opensearch-project/documentation-website/issues/7700), and will be used to [generate the OpenSearch server API itself](https://github.com/opensearch-project/OpenSearch/issues/3090). + +### Working in this Repo + +Because of our legacy, the specification is produced by reading the [OpenSearch documentation](https://opensearch.org/docs/latest/) and reverse-engineering code. Thus, a good place to start contributing to this repo is to [identify a missing API and to add it](https://github.com/opensearch-project/opensearch-api-specification/issues/168) by following the [developer guide](DEVELOPER_GUIDE.md). This repo also contains a set of [test tools](TESTING_GUIDE.md) that ensure the correctness of this API and infrastructure that evaluates the gap between the OpenSearch server and this API spec by comparing REST routes registered in a running OpenSearch to the list of the APIs in this repo. You can see the latest API and test coverage numbers in the comments automatically added to any [recently merged pull requests](https://github.com/opensearch-project/opensearch-api-specification/pulls?q=is%3Apr+is%3Aclosed). ## Project Resources From 2a39edde31abf8ae885366a6abb6c7a48c481ba7 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 6 Nov 2024 18:28:23 -0500 Subject: [PATCH 71/73] Ensure correct OpenAPI 3.1.0 spec. (#646) * Added Python and Ruby spec validators. Signed-off-by: dblock * Add empty descriptions where missing and remove descriptions next to refs. Signed-off-by: dblock * Replaced the validator compatible with OpenAPI 3.1. Signed-off-by: dblock * Fixed incorrect style. Signed-off-by: dblock * Added missing require. Signed-off-by: dblock * Fix: schema null is not supported. Signed-off-by: dblock * Removed unused required property. Signed-off-by: dblock * Fix: added missing schema. Signed-off-by: dblock * Fix: removed incorrect defaults. Signed-off-by: dblock * Fix: added missing required. Signed-off-by: dblock * Fix: removed incorrect required. Signed-off-by: dblock * Removed incorrect externalDocs. Signed-off-by: dblock * Document the ruby spec validator. Signed-off-by: dblock * Fix: removed redundant content types. Signed-off-by: dblock * Improve displaying of errors. Signed-off-by: dblock * Normalize keys to be compatible with OpenAPI 3.1. Signed-off-by: dblock * Removed superseeded operations that have unresolved parameters. Signed-off-by: dblock * Fix: don't reduce error payload, allows to verify schema. Signed-off-by: dblock * Fix: flow framework schema tests. Signed-off-by: dblock * Fix: if no decision is made there's no weight ranking. Signed-off-by: dblock * Fix: wrap indexing errors. Signed-off-by: dblock * Added verbose logging of the story being evaluated. Signed-off-by: dblock * Fix: security API test error responses. Signed-off-by: dblock * Fix: duration ref. Signed-off-by: dblock * Fix typo in property. Signed-off-by: dblock * Fix superseeded operation parameters. Signed-off-by: dblock * Fix: neural search error response. Signed-off-by: dblock * Normalize fields. .replaceAll('::', '___') .replaceAll('@', '__') .replaceAll(':', '_') Signed-off-by: dblock * Re-added default values into the schema. Signed-off-by: dblock * Disallowed characters are replaced with ___. Signed-off-by: dblock --------- Signed-off-by: dblock --- .cspell | 2 + ...lidate-spec.yml => validate-spec-lint.yml} | 0 .github/workflows/validate-spec-py.yml | 34 ++ .github/workflows/validate-spec-ruby.yml | 30 ++ CHANGELOG.md | 12 + DEVELOPER_GUIDE.md | 30 +- spec/_global_parameters.yaml | 6 +- spec/_superseded_operations.yaml | 30 +- spec/namespaces/asynchronous_search.yaml | 2 + spec/namespaces/cat.yaml | 69 ++- spec/namespaces/flow_framework.yaml | 148 +++--- spec/namespaces/indices.yaml | 4 +- spec/namespaces/ism.yaml | 6 +- spec/namespaces/observability.yaml | 3 +- spec/namespaces/ppl.yaml | 3 - spec/namespaces/search_pipeline.yaml | 1 + spec/namespaces/security.yaml | 141 ++++-- spec/namespaces/sql.yaml | 9 - spec/schemas/cluster.allocation_explain.yaml | 1 - spec/schemas/flow_framework.common.yaml | 35 +- spec/schemas/flow_framework.errors.yaml | 216 ++++----- spec/schemas/indices._common.yaml | 7 + spec/schemas/insights._common.yaml | 2 +- spec/schemas/observability._common.yaml | 10 +- spec/schemas/query._common.yaml | 8 + spec/schemas/security._common.yaml | 16 + tests/default/flow_framework/template.yaml | 2 + tests/default/security/api/nodesdn.yaml | 65 --- .../ml/ingest/pipeline/neural_search.yaml | 17 - .../ml/ingest/pipeline/search_by_image.yaml | 26 + tools/src/linter/SchemasValidator.ts | 12 +- tools/src/merger/OpenApiMerger.ts | 55 +++ tools/src/tester/ChapterReader.ts | 5 +- tools/src/tester/TestRunner.ts | 6 +- tools/src/tester/test.ts | 2 +- tools/src/validate-spec-py/Pipfile | 12 + tools/src/validate-spec-py/Pipfile.lock | 455 ++++++++++++++++++ tools/src/validate-spec-py/validate.py | 20 + tools/src/validate-spec-ruby/Gemfile | 3 + tools/src/validate-spec-ruby/Gemfile.lock | 22 + tools/src/validate-spec-ruby/validate.rb | 20 + .../extractor/opensearch/expected_1.3.yaml | 61 +-- .../extractor/opensearch/expected_2.0.yaml | 71 +-- .../fixtures/merger/animals/expected.yaml | 76 +-- tools/tests/tester/ChapterReader.test.ts | 29 ++ tools/tests/tester/MergedOpenApiSpec.test.ts | 10 +- tools/tests/tester/helpers.ts | 2 +- 47 files changed, 1295 insertions(+), 501 deletions(-) rename .github/workflows/{validate-spec.yml => validate-spec-lint.yml} (100%) create mode 100644 .github/workflows/validate-spec-py.yml create mode 100644 .github/workflows/validate-spec-ruby.yml delete mode 100644 tests/default/security/api/nodesdn.yaml create mode 100644 tests/plugins/ml/ingest/pipeline/search_by_image.yaml create mode 100644 tools/src/validate-spec-py/Pipfile create mode 100644 tools/src/validate-spec-py/Pipfile.lock create mode 100644 tools/src/validate-spec-py/validate.py create mode 100644 tools/src/validate-spec-ruby/Gemfile create mode 100644 tools/src/validate-spec-ruby/Gemfile.lock create mode 100644 tools/src/validate-spec-ruby/validate.rb diff --git a/.cspell b/.cspell index 274d2da09..eb1d955ba 100644 --- a/.cspell +++ b/.cspell @@ -63,6 +63,7 @@ geotile gsub Gsub haasephonetik +hashcode heteroscedastic hnsw homoscedastic @@ -134,6 +135,7 @@ Oversample performanceanalyzer permissionsinfo pipefail +pipenv preconfigure preconfigured prefilter diff --git a/.github/workflows/validate-spec.yml b/.github/workflows/validate-spec-lint.yml similarity index 100% rename from .github/workflows/validate-spec.yml rename to .github/workflows/validate-spec-lint.yml diff --git a/.github/workflows/validate-spec-py.yml b/.github/workflows/validate-spec-py.yml new file mode 100644 index 000000000..9e33a5aaf --- /dev/null +++ b/.github/workflows/validate-spec-py.yml @@ -0,0 +1,34 @@ +name: Validate Spec (Python) + +on: [pull_request,push] + +jobs: + validate-spec-py: + runs-on: ubuntu-latest + steps: + - name: Checkout the repo + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: Build + run: npm ci && npm run merge + + - name: Set Up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install Dependencies + working-directory: tools/src/validate-spec-py + run: | + pip install --user pipenv + pipenv install + + - name: Validate Spec + working-directory: tools/src/validate-spec-py + run: | + pipenv run python validate.py ../../../build/opensearch-openapi.yaml diff --git a/.github/workflows/validate-spec-ruby.yml b/.github/workflows/validate-spec-ruby.yml new file mode 100644 index 000000000..de0e1e200 --- /dev/null +++ b/.github/workflows/validate-spec-ruby.yml @@ -0,0 +1,30 @@ +name: Validate Spec (Ruby) + +on: [pull_request, push] + +jobs: + validate-spec-ruby: + runs-on: ubuntu-latest + env: + BUNDLE_GEMFILE: ${{ github.workspace }}/tools/src/validate-spec-ruby/Gemfile + steps: + - name: Checkout the repo + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v3 + with: + node-version: '20' + + - name: Build + run: npm ci && npm run merge + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' + bundler-cache: true + + - name: Validate Spec + working-directory: tools/src/validate-spec-ruby + run: | + bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc71eaeb..811c2cb21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,22 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Added - Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) +- Added Python and Ruby spec validators ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Added verbose output of the story being evaluated ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) +### Fixed +- Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Disallowed characters (`::`, `@`, and `:`) in key names are replaced with `___` on merge ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Added missing `required` to `path` parameters ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Added missing `schema` parent to response types ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Removed invalid `externalDocs` from `flow_framework.create/update::query.use_case` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Fixed incorrect `style` in `indices.get_mapping::query.index` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Removed invalid `required` from `ppl` responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Added schema for security API error responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) + ## [0.1.0] - 2024-10-25 ### Added diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b46319eb6..7ff326560 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -26,7 +26,9 @@ - [Comment on PR](#comment-on-pr) - [Test Tools (Unit)](#test-tools-unit) - [Test Tools (Integration)](#test-tools-integration) - - [Validate Spec](#validate-spec) + - [Validate Spec (Lint)](#validate-spec-lint) + - [Validate Spec (Python)](#validate-spec-python) + - [Validate Spec (Ruby)](#validate-spec-ruby) # Developer Guide @@ -375,6 +377,30 @@ This workflow runs on PRs to invoke the [tools' unit tests](tools/tests), upload This workflow runs on PRs to invoke the [tools' integration tests](tools/tests) that require a running instance of OpenSearch to ensure there are no breakages in behavior. -### [Validate Spec](.github/workflows/validate-spec.yml) +### [Validate Spec (Lint)](.github/workflows/validate-spec-lint.yml) This workflow runs on PRs to invoke the [spec linter](#spec-linter) and ensure the multi-file spec is correct and follows the design guidelines. + +### [Validate Spec (Python)](.github/workflows/validate-spec-py.yml) + +This workflow runs on PRs to invoke the [Python openapi-spec-validator](https://pypi.org/project/openapi-spec-validator/) to ensure that the resulting spec can be loaded by Python tools. + +You can run the validator locally as follows after installing [pipenv](https://pipenv.pypa.io/en/latest/installation.html). + +``` +cd tools/src/validate-spec-py +pipenv install +npm run merge ; pipenv run python validate.py ../../../build/opensearch-openapi.yaml +``` + +### [Validate Spec (Ruby)](.github/workflows/validate-spec-ruby.yml) + +This workflow runs on PRs to invoke the Ruby [Json Schemer](https://github.com/davishmcclurg/json_schemer/) to ensure that the resulting spec can be loaded by Ruby tools. + +You can run the validator locally as follows. + +``` +cd tools/src/validate-spec-ruby +bundle install +npm run merge ; bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml +``` diff --git a/spec/_global_parameters.yaml b/spec/_global_parameters.yaml index 91ddbfe8f..160d1d241 100644 --- a/spec/_global_parameters.yaml +++ b/spec/_global_parameters.yaml @@ -10,21 +10,21 @@ components: description: Whether to pretty format the returned JSON response. schema: type: boolean - default: false + default: false human: name: human in: query description: Whether to return human readable values for statistics. schema: type: boolean - default: true + default: true error_trace: name: error_trace in: query description: Whether to include the stack trace of returned errors. schema: type: boolean - default: false + default: false source: name: source in: query diff --git a/spec/_superseded_operations.yaml b/spec/_superseded_operations.yaml index 80e556067..fbc54730f 100644 --- a/spec/_superseded_operations.yaml +++ b/spec/_superseded_operations.yaml @@ -201,7 +201,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PUT /_opendistro/_ism/policies/{policyID}: - superseded_by: /_plugins/_ism/policies/{policyID} + superseded_by: /_plugins/_ism/policies/{policy_id} operations: - GET - HEAD @@ -240,7 +240,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml operations: - GET /_opendistro/_knn/{nodeId}/stats/{stat}: - superseded_by: /_plugins/_knn/{nodeId}/stats/{stat} + superseded_by: /_plugins/_knn/{node_id}/stats/{stat} operations: - GET /_opendistro/_performanceanalyzer/_agent/{redirectEndpoint}: @@ -354,22 +354,22 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PUT /_opendistro/_rollup/jobs/{rollupID}: - superseded_by: /_plugins/_rollup/jobs/{rollupID} + superseded_by: /_plugins/_rollup/jobs/{id} operations: - GET - HEAD - PUT - DELETE /_opendistro/_rollup/jobs/{rollupID}/_explain: - superseded_by: /_plugins/_rollup/jobs/{rollupID}/_explain + superseded_by: /_plugins/_rollup/jobs/{id}/_explain operations: - GET /_opendistro/_rollup/jobs/{rollupID}/_start: - superseded_by: /_plugins/_rollup/jobs/{rollupID}/_start + superseded_by: /_plugins/_rollup/jobs/{id}/_start operations: - POST /_opendistro/_rollup/jobs/{rollupID}/_stop: - superseded_by: /_plugins/_rollup/jobs/{rollupID}/_stop + superseded_by: /_plugins/_rollup/jobs/{id}/_stop operations: - POST /_opendistro/_security/api/_upgrade_check/: @@ -401,7 +401,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PATCH /_opendistro/_security/api/actiongroups/{name}: - superseded_by: /_plugins/_security/api/actiongroups/{name} + superseded_by: /_plugins/_security/api/actiongroups/{action_group} operations: - GET - PUT @@ -433,14 +433,14 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PATCH /_opendistro/_security/api/internalusers/{name}: - superseded_by: /_plugins/_security/api/internalusers/{name} + superseded_by: /_plugins/_security/api/internalusers/{username} operations: - GET - PUT - PATCH - DELETE /_opendistro/_security/api/internalusers/{name}/authtoken: - superseded_by: /_plugins/_security/api/internalusers/{name}/authtoken + superseded_by: /_plugins/_security/api/internalusers/{username}/authtoken operations: - POST /_opendistro/_security/api/migrate: @@ -457,7 +457,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PATCH /_opendistro/_security/api/roles/{name}: - superseded_by: /_plugins/_security/api/roles/{name} + superseded_by: /_plugins/_security/api/roles/{role} operations: - GET - PUT @@ -469,7 +469,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PATCH /_opendistro/_security/api/rolesmapping/{name}: - superseded_by: /_plugins/_security/api/rolesmapping/{name} + superseded_by: /_plugins/_security/api/rolesmapping/{role} operations: - GET - PUT @@ -519,7 +519,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml - GET - PATCH /_opendistro/_security/api/tenants/{name}: - superseded_by: /_plugins/_security/api/tenants/{name} + superseded_by: /_plugins/_security/api/tenants/{tenant} operations: - GET - PUT @@ -529,14 +529,14 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml superseded_by: /_plugins/_security/api/user operations: - GET -/_opendistro/_security/api/user/{name}: - superseded_by: /_plugins/_security/api/user/{name} +/_opendistro/_security/api/user/{username}: + superseded_by: /_plugins/_security/api/user/{username} operations: - GET - PUT - DELETE /_opendistro/_security/api/user/{name}/authtoken: - superseded_by: /_plugins/_security/api/user/{name}/authtoken + superseded_by: /_plugins/_security/api/user/{username}/authtoken operations: - POST /_opendistro/_security/api/validate: diff --git a/spec/namespaces/asynchronous_search.yaml b/spec/namespaces/asynchronous_search.yaml index 863cd33d6..9153257a9 100644 --- a/spec/namespaces/asynchronous_search.yaml +++ b/spec/namespaces/asynchronous_search.yaml @@ -90,11 +90,13 @@ components: in: path schema: type: string + required: true asynchronous_search.delete::path.id: name: id in: path schema: type: string + required: true requestBodies: asynchronous_search.search: content: diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index 9e6f27684..c89368e5b 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -792,7 +792,8 @@ components: cat.aliases@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -801,7 +802,8 @@ components: cat.all_pit_segments@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -810,7 +812,8 @@ components: cat.allocation@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -819,7 +822,8 @@ components: cat.cluster_manager@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -828,7 +832,8 @@ components: cat.count@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -837,7 +842,8 @@ components: cat.fielddata@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -846,7 +852,8 @@ components: cat.health@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -875,7 +882,8 @@ components: cat.indices@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -899,7 +907,8 @@ components: cat.master@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -908,7 +917,8 @@ components: cat.nodeattrs@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -917,7 +927,8 @@ components: cat.nodes@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -926,7 +937,8 @@ components: cat.pending_tasks@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -935,7 +947,8 @@ components: cat.pit_segments@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -944,7 +957,8 @@ components: cat.plugins@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -953,7 +967,8 @@ components: cat.recovery@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -962,7 +977,8 @@ components: cat.repositories@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -971,7 +987,8 @@ components: cat.segment_replication@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -980,7 +997,8 @@ components: cat.segments@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -989,7 +1007,8 @@ components: cat.shards@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -998,7 +1017,8 @@ components: cat.snapshots@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -1007,7 +1027,8 @@ components: cat.tasks@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -1016,7 +1037,8 @@ components: cat.templates@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array @@ -1025,7 +1047,8 @@ components: cat.thread_pool@200: content: text/plain: - type: string + schema: + type: string application/json: schema: type: array diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index feed463e6..7d3616220 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -302,8 +302,6 @@ components: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' flow_framework.create::query.use_case: description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. - externalDocs: - url: https://opensearch.org/docs/latest/automating-configurations/workflow-templates/#supported-workflow-templates in: query name: use_case schema: @@ -316,8 +314,6 @@ components: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/All' flow_framework.update::query.use_case: description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. - externalDocs: - url: https://opensearch.org/docs/latest/automating-configurations/workflow-templates/#supported-workflow-templates in: query name: use_case schema: @@ -400,36 +396,39 @@ components: flow_framework.create@403: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.create@400: description: Bad Request - Multiple possible reasons content: application/json: - oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNameRequiredError' + schema: + anyOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNameRequiredError' flow_framework.update@400: description: Bad Request - Multiple possible reasons content: application/json: - oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' + schema: + anyOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidTemplateVersionError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/UnsupportedFieldUpdateError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowParsingError' flow_framework.update@201: content: application/json: @@ -444,121 +443,148 @@ components: flow_framework.delete@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkDeleteResponse' flow_framework.get@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkGetResponse' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkGetResponse' flow_framework.update@404: description: Template Not Found Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' flow_framework.delete@400: description: Work Flow Id Null Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' flow_framework.delete@403: description: Flow Framework API Disabled Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.get@404: description: Template Not Found Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' flow_framework.get_steps@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowStep' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSteps' flow_framework.get_steps@403: description: Flow Framework API Disabled Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.get_steps@400: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowStepsRetrieveError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowStepsRetrieveError' flow_framework.get_status@403: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.get_status@404: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' flow_framework.get_status@200: content: application/json: - oneOf: - - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusFullResponse' - - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusDefaultResponse' + schema: + anyOf: + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusFullResponse' + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusDefaultResponse' flow_framework.deprovision@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowIDResponse' flow_framework.provision@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowIDResponse' flow_framework.deprovision@202: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningError' flow_framework.deprovision@403: content: application/json: - oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningForbiddenError' + schema: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningForbiddenError' flow_framework.deprovision@404: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' flow_framework.provision@403: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.provision@400: content: application/json: - oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestBodyParsingFailedError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidRequestBodyFieldError' - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DuplicateKeyError' + schema: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestBodyParsingFailedError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidRequestBodyFieldError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DuplicateKeyError' flow_framework.search@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' flow_framework.search@403: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.search@408: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' flow_framework.search@400: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' flow_framework.search_state@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchStateResponse' + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchStateResponse' flow_framework.search_state@403: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' flow_framework.search_state@408: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' flow_framework.search_state@400: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' \ No newline at end of file + schema: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' \ No newline at end of file diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index cbe075a6d..fcf676adf 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2404,7 +2404,7 @@ components: content: application/json: schema: - $ref: '../schemas/indices._common.yaml#/components/schemas/IndexErrorCause' + $ref: '../schemas/indices._common.yaml#/components/schemas/IndexError' indices.delete_alias@200: content: application/json: @@ -3965,7 +3965,7 @@ components: To target all data streams and indices, omit this parameter or use `*` or `_all`. schema: $ref: '../schemas/_common.yaml#/components/schemas/Indices' - style: simple + style: form indices.get_mapping::query.local: in: query name: local diff --git a/spec/namespaces/ism.yaml b/spec/namespaces/ism.yaml index 334bc69f7..9e91d84c7 100644 --- a/spec/namespaces/ism.yaml +++ b/spec/namespaces/ism.yaml @@ -178,7 +178,7 @@ paths: '200': $ref: '#/components/responses/ism.explain_policy@200' post: - operationId: ism.explain_policy.0 + operationId: ism.explain_policy.1 x-operation-group: ism.explain_policy description: Gets the currently applied policy on indices. externalDocs: @@ -190,7 +190,7 @@ paths: $ref: '#/components/responses/ism.explain_policy@200' /_plugins/_ism/explain/{index}: get: - operationId: ism.explain_policy.1 + operationId: ism.explain_policy.3 x-operation-group: ism.explain_policy description: Gets the currently applied policy on indices. externalDocs: @@ -203,7 +203,7 @@ paths: '200': $ref: '#/components/responses/ism.explain_policy@200' post: - operationId: ism.explain_policy.1 + operationId: ism.explain_policy.4 x-operation-group: ism.explain_policy description: Gets the currently applied policy on indices. externalDocs: diff --git a/spec/namespaces/observability.yaml b/spec/namespaces/observability.yaml index a8553dbe1..62cca3dfd 100644 --- a/spec/namespaces/observability.yaml +++ b/spec/namespaces/observability.yaml @@ -150,7 +150,6 @@ components: type: object additionalProperties: type: string - example: OK observability.delete_object@404: description: Not Found content: @@ -168,7 +167,7 @@ components: type: object additionalProperties: type: string - example: OK + # example: OK observability.delete_objects@404: description: Not Found content: diff --git a/spec/namespaces/ppl.yaml b/spec/namespaces/ppl.yaml index 0794fd32d..02ef9e2bb 100644 --- a/spec/namespaces/ppl.yaml +++ b/spec/namespaces/ppl.yaml @@ -150,16 +150,13 @@ components: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/ExplainResponse' - required: true ppl.get_stats@200: content: text/plain: schema: type: string - required: true ppl.post_stats@200: content: text/plain: schema: type: string - required: true \ No newline at end of file diff --git a/spec/namespaces/search_pipeline.yaml b/spec/namespaces/search_pipeline.yaml index 05d16fbdd..a2b5bd5fd 100644 --- a/spec/namespaces/search_pipeline.yaml +++ b/spec/namespaces/search_pipeline.yaml @@ -106,6 +106,7 @@ components: description: Comma-separated list of search pipeline ids. Wildcards supported. schema: type: string + required: true search_pipeline.get::query.cluster_manager_timeout: name: cluster_manager_timeout in: query diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 5900516f5..77dc0c6f6 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -1558,7 +1558,8 @@ components: security.cache@501: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.change_password@200: content: application/json: @@ -1567,7 +1568,8 @@ components: security.change_password@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.config_upgrade_check@200: content: application/json: @@ -1581,7 +1583,8 @@ components: security.config_upgrade_perform@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.create_action_group@200: content: application/json: @@ -1600,7 +1603,8 @@ components: security.create_allowlist@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.create_role@200: content: application/json: @@ -1634,7 +1638,8 @@ components: security.create_tenant@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.create_update_tenancy_config@200: content: application/json: @@ -1643,7 +1648,8 @@ components: security.create_update_tenancy_config@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.create_user@200: content: application/json: @@ -1672,11 +1678,13 @@ components: security.delete_distinguished_name@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.delete_distinguished_name@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.delete_role@200: content: application/json: @@ -1695,7 +1703,8 @@ components: security.delete_tenant@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.delete_user@200: content: application/json: @@ -1719,9 +1728,11 @@ components: security.generate_obo_token@400: content: text/plain: - type: string + schema: + type: string application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.generate_user_token@200: content: application/json: @@ -1730,11 +1741,13 @@ components: security.generate_user_token@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.generate_user_token_legacy@501: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_account_details@200: content: application/json: @@ -1758,7 +1771,8 @@ components: security.get_allowlist@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_audit_configuration@200: content: application/json: @@ -1777,7 +1791,8 @@ components: security.get_all_certificates@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_all_certificates@500: content: application/json: @@ -1791,15 +1806,18 @@ components: security.get_node_certificates@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_certificates@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_certificates@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_node_certificates@500: content: application/json: @@ -1829,11 +1847,13 @@ components: security.get_distinguished_name@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_distinguished_name@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_distinguished_names@200: content: application/json: @@ -1843,11 +1863,13 @@ components: description: Show nodesDn setting for given cluster. content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_distinguished_names@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_permissions_info@200: content: application/json: @@ -1901,7 +1923,8 @@ components: security.get_tenancy_config@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_tenants@200: content: application/json: @@ -1910,7 +1933,8 @@ components: security.get_tenants@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.get_user@200: content: application/json: @@ -1944,7 +1968,8 @@ components: security.migrate@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_action_group@200: content: application/json: @@ -1963,7 +1988,8 @@ components: security.patch_allowlist@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_audit_configuration@200: content: application/json: @@ -1977,7 +2003,8 @@ components: security.patch_configuration@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_distinguished_name@200: content: application/json: @@ -1986,11 +2013,13 @@ components: security.patch_distinguished_name@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_distinguished_name@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_distinguished_names@200: content: application/json: @@ -1999,11 +2028,13 @@ components: security.patch_distinguished_names@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_distinguished_names@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_role@200: content: application/json: @@ -2012,7 +2043,8 @@ components: security.patch_role@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_role_mapping@200: content: application/json: @@ -2021,7 +2053,8 @@ components: security.patch_role_mapping@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_role_mappings@200: content: application/json: @@ -2030,7 +2063,8 @@ components: security.patch_role_mappings@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_roles@200: content: application/json: @@ -2039,7 +2073,8 @@ components: security.patch_roles@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_tenant@200: content: application/json: @@ -2048,7 +2083,8 @@ components: security.patch_tenant@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_tenants@200: content: application/json: @@ -2057,7 +2093,8 @@ components: security.patch_tenants@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.patch_user@200: content: application/json: @@ -2086,11 +2123,13 @@ components: security.reload_http_certificates@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.reload_http_certificates@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.reload_transport_certificates@200: content: application/json: @@ -2099,11 +2138,13 @@ components: security.reload_transport_certificates@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.reload_transport_certificates@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.tenant_info@200: content: application/json: @@ -2112,9 +2153,11 @@ components: security.tenant_info@403: content: text/plain: - type: string + schema: + type: string application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.tenant_info@500: content: application/json: @@ -2133,7 +2176,8 @@ components: security.update_configuration@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.update_distinguished_name@200: content: application/json: @@ -2142,11 +2186,13 @@ components: security.update_distinguished_name@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.update_distinguished_name@403: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.validate@200: content: application/json: @@ -2155,7 +2201,8 @@ components: security.validate@400: content: application/json: - schema: null + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Error' security.who_am_i@200: content: application/json: diff --git a/spec/namespaces/sql.yaml b/spec/namespaces/sql.yaml index a54c6cc39..053ebead0 100644 --- a/spec/namespaces/sql.yaml +++ b/spec/namespaces/sql.yaml @@ -177,31 +177,26 @@ components: anyOf: - $ref: '../schemas/sql._common.yaml#/components/schemas/SqlSettingsPlain' - $ref: '../schemas/sql._common.yaml#/components/schemas/SqlSettings' - required: true sql.query: content: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/Query' - required: true sql.explain: content: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/Explain' - required: true sql.close: content: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/SqlClose' - required: true sql.post_stats: content: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/Stats' - required: true responses: sql.settings@200: content: @@ -218,22 +213,18 @@ components: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/ExplainResponse' - required: true sql.close@200: content: application/json: schema: $ref: '../schemas/sql._common.yaml#/components/schemas/SqlCloseResponse' - required: true sql.get_stats@200: content: text/plain: schema: type: string - required: true sql.post_stats@200: content: text/plain: schema: type: string - required: true \ No newline at end of file diff --git a/spec/schemas/cluster.allocation_explain.yaml b/spec/schemas/cluster.allocation_explain.yaml index f7141a328..97b36f6ef 100644 --- a/spec/schemas/cluster.allocation_explain.yaml +++ b/spec/schemas/cluster.allocation_explain.yaml @@ -177,7 +177,6 @@ components: - node_id - node_name - transport_address - - weight_ranking AllocationStore: type: object properties: diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index d3ab4ae9b..2c3678921 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -11,6 +11,12 @@ components: type: string WorkflowID: type: string + WorkflowIDResponse: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow WorkflowStepName: type: string Provision: @@ -99,6 +105,8 @@ components: properties: name: type: string + version: + $ref: '#/components/schemas/version' description: type: string use_case: @@ -111,6 +119,8 @@ components: type: integer last_updated_time: type: integer + last_provisioned_time: + type: number version: type: object description: A key-value map with two fields, template, which identifies the template version, and compatibility, which identifies a list of minimum required OpenSearch versions. @@ -135,6 +145,9 @@ components: failed: type: integer description: The number of shards where the deletion failed. + skipped: + type: integer + description: The number of shards where the deletion was skipped. query: type: object description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`. @@ -237,9 +250,11 @@ components: total: $ref: '#/components/schemas/total' max_score: - type: number - format: float - description: The maximum score of the search hits. + oneOf: + - type: 'null' + - type: number + format: float + description: The maximum score of the search hits. hits: type: array items: @@ -250,13 +265,19 @@ components: total: $ref: '#/components/schemas/total' max_score: - type: number - format: float - description: The maximum score of the search hits. + oneOf: + - type: 'null' + - type: number + format: float + description: The maximum score of the search hits. hits: type: array items: $ref: '#/components/schemas/StateItems' + WorkflowSteps: + type: object + additionalProperties: + $ref: '#/components/schemas/WorkflowStep' WorkflowStep: type: object properties: @@ -275,6 +296,8 @@ components: items: type: string description: The list of plugins required for the workflow step + timeout: + $ref: '_common.yaml#/components/schemas/Duration' WorkFlowStatusDefaultResponse: type: object properties: diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 879cf7fe5..c91ed0e1d 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -7,65 +7,53 @@ paths: {} components: schemas: FlowFrameworkAPIDisabledError: - content: - application/json: - type: object - properties: - error: - type: string - example: This API is disabled. To enable it, set [flow_framework.enabled] to true. - status: - type: integer + type: object + properties: + error: + type: string + # example: This API is disabled. To enable it, set [flow_framework.enabled] to true. + status: + type: integer InvalidParameterError: - content: - application/json: - type: object - properties: - error: - type: string - example: Only the parameters [param1, param2] are permitted unless the provision parameter is set to true. - status: - type: integer + type: object + properties: + error: + type: string + # example: Only the parameters [param1, param2] are permitted unless the provision parameter is set to true. + status: + type: integer ConflictError: - content: - application/json: - type: object - properties: - error: - type: string - example: You cannot use both the 'provision_workflow' and 'update_workflow_fields' parameters in the same request. - status: - type: integer + type: object + properties: + error: + type: string + # example: You cannot use both the 'provision_workflow' and 'update_workflow_fields' parameters in the same request. + status: + type: integer MissingParameterError: - content: - application/json: - type: object - properties: - error: - type: string - example: You cannot use the 'reprovision_workflow' parameter to create a new template. - status: - type: integer + type: object + properties: + error: + type: string + # example: You cannot use the 'reprovision_workflow' parameter to create a new template. + status: + type: integer ParameterConflictError: - content: - application/json: - type: object - properties: - error: - type: string - example: You cannot use the 'reprovision_workflow' and 'use_case' parameters in the same request. - status: - type: integer + type: object + properties: + error: + type: string + # example: You cannot use the 'reprovision_workflow' and 'use_case' parameters in the same request. + status: + type: integer WorkFlowIdNullError: - content: - application/json: - type: object - properties: - error: - type: string - example: Workflow ID can not be null - status: - type: integer + type: object + properties: + error: + type: string + # example: Workflow ID can not be null + status: + type: integer DuplicateKeyError: type: object properties: @@ -75,7 +63,7 @@ components: status: type: integer description: HTTP status code for the error. - example: 400 + # example: 400 InvalidRequestBodyFieldError: type: object properties: @@ -94,27 +82,23 @@ components: status: type: integer description: HTTP status code for the error. - example: 400 + # example: 400 WorkflowSaveError: - content: - application/json: - type: object - properties: - error: - type: string - example: Failed to save workflow state - status: - type: integer + type: object + properties: + error: + type: string + # example: Failed to save workflow state + status: + type: integer MaxWorkflowsLimitError: - content: - application/json: - type: object - properties: - error: - type: string - example: Maximum workflows limit reached 50 - code: - type: integer + type: object + properties: + error: + type: string + # example: Maximum workflows limit reached 50 + code: + type: integer TemplateNameRequiredError: type: object properties: @@ -122,59 +106,49 @@ components: type: string description: Error message when the template name is missing. TemplateNotFoundError: - content: - application/json: - type: object - properties: - error: - type: string - example: Failed to retrieve template (12345) from global context. - code: - type: integer + type: object + properties: + error: + type: string + # example: Failed to retrieve template (12345) from global context. + code: + type: integer InvalidTemplateVersionError: - content: - application/json: - type: object - properties: - error: - type: string - description: Error message when the template version is invalid or missing. - example: Unable to parse field [version] in a version object. + type: object + properties: + error: + type: string + description: Error message when the template version is invalid or missing. + # example: Unable to parse field [version] in a version object. UnsupportedFieldUpdateError: - content: - application/json: - type: object - properties: - error: - type: string - description: Error message when trying to update an unsupported field in a template. - example: You can not update the field [fieldName] without updating the whole template. + type: object + properties: + error: + type: string + description: Error message when trying to update an unsupported field in a template. + # example: You can not update the field [fieldName] without updating the whole template. WorkflowParsingError: - content: - application/json: - type: object - properties: - error: - type: string - description: Error message when workflow parsing fails. - example: Unable to parse field [workflow] in a template object. + type: object + properties: + error: + type: string + description: Error message when workflow parsing fails. + # example: Unable to parse field [workflow] in a template object. WorkflowStepsRetrieveError: - content: - application/json: - type: object - properties: - error: - type: string - example: Failed to retrieve workflow step json. - code: - type: integer + type: object + properties: + error: + type: string + # example: Failed to retrieve workflow step json. + code: + type: integer DeprovisioningError: type: object properties: error: type: string description: Describes the deprovisioning error and identifies resources that were not deprovisioned - example: Failed to deprovision some resources [connector_id Lw7PX4wBfVtHp98y06wV]. + # example: Failed to deprovision some resources [connector_id Lw7PX4wBfVtHp98y06wV]. required: - error DeprovisioningForbiddenError: @@ -183,7 +157,7 @@ components: error: type: string description: Describes the resources that require the allow_delete parameter for deprovisioning - example: These resources require the allow_delete parameter to deprovision [index_name my-index]. + # example: These resources require the allow_delete parameter to deprovision [index_name my-index]. required: - error RequestTimeoutError: @@ -195,15 +169,15 @@ components: status: type: integer description: HTTP status code for the error. - example: 408 + # example: 408 BadRequestError: type: object properties: error: type: string description: Error message when the request body or parameters are invalid. - example: Invalid request body or query parameters. + # example: Invalid request body or query parameters. status: type: integer description: HTTP status code for the error. - example: 400 \ No newline at end of file + # example: 400 \ No newline at end of file diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index 2e3763cc4..10386d1ea 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -983,6 +983,13 @@ components: - mappings - order - settings + IndexError: + type: object + properties: + error: + $ref: '#/components/schemas/IndexErrorCause' + status: + type: number IndexErrorCause: type: object properties: diff --git a/spec/schemas/insights._common.yaml b/spec/schemas/insights._common.yaml index b4a700360..66b0bbe47 100644 --- a/spec/schemas/insights._common.yaml +++ b/spec/schemas/insights._common.yaml @@ -30,7 +30,7 @@ components: items: type: object $ref: '#/components/schemas/TaskResourceUsages' - # eslint-disable-next-line @cspell/spellchecker + query_hashcode: type: string description: The hash code of the query. diff --git a/spec/schemas/observability._common.yaml b/spec/schemas/observability._common.yaml index fc8018f55..08b7fbce6 100644 --- a/spec/schemas/observability._common.yaml +++ b/spec/schemas/observability._common.yaml @@ -218,7 +218,7 @@ components: $ref: '#/components/schemas/ErrorResponse' status: type: integer - example: 404 + # example: 404 required: - error - status @@ -232,10 +232,10 @@ components: $ref: '#/components/schemas/RootCause' type: type: string - example: status_exception + # example: status_exception reason: type: string - example: 'ObservabilityObject {objectId} not found' + # example: 'ObservabilityObject {objectId} not found' required: - reason - root_cause @@ -246,10 +246,10 @@ components: properties: type: type: string - example: status_exception + # example: status_exception reason: type: string - example: 'ObservabilityObject {objectId} not found' + # example: 'ObservabilityObject {objectId} not found' required: - reason - type diff --git a/spec/schemas/query._common.yaml b/spec/schemas/query._common.yaml index 40277b8b0..ef4e315f4 100644 --- a/spec/schemas/query._common.yaml +++ b/spec/schemas/query._common.yaml @@ -71,6 +71,14 @@ components: - error ErrorResponse: + type: object + properties: + error: + $ref: '#/components/schemas/Error' + status: + type: number + + Error: type: object properties: root_cause: diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index d98618758..fde8ffb39 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -454,6 +454,22 @@ components: items: type: string + Error: + type: object + properties: + status: + type: string + enum: + - BAD_REQUEST + - CONFLICT + - FORBIDDEN + - INTERNAL_SERVER_ERROR + - NOT_FOUND + - NOT_IMPLEMENTED + - UNAUTHORIZED + message: + type: string + InternalServerError: type: object properties: diff --git a/tests/default/flow_framework/template.yaml b/tests/default/flow_framework/template.yaml index c7ac780c9..84ecd8638 100644 --- a/tests/default/flow_framework/template.yaml +++ b/tests/default/flow_framework/template.yaml @@ -7,6 +7,8 @@ epilogues: status: [200, 404] parameters: workflow_id: ${create_flow_framework.test_workflow_id} +warnings: + multiple-paths-detected: false chapters: - synopsis: Create workflow with openAI model. id: create_flow_framework diff --git a/tests/default/security/api/nodesdn.yaml b/tests/default/security/api/nodesdn.yaml deleted file mode 100644 index 43b3991d4..000000000 --- a/tests/default/security/api/nodesdn.yaml +++ /dev/null @@ -1,65 +0,0 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml - -description: Test nodesdn endpoints. -distributions: - excluded: - - amazon-managed - - amazon-serverless - -# ADMIN-CERT only. These tests require explicit rest api admin privileges. -# The setting `plugins. security. nodes_dn_dynamic_config_enabled` must be enabled. -chapters: - - synopsis: Get distinguished names. - path: /_plugins/_security/api/nodesdn - method: GET - parameters: - show_all: true - response: - status: 400 - - synopsis: Patch distinguished names. - path: /_plugins/_security/api/nodesdn - method: PATCH - request: - payload: - - op: replace - path: /cluster1/nodes_dn/0 - value: [''] - response: - status: 400 - - synopsis: Create distinguished name. - path: /_plugins/_security/api/nodesdn/{cluster_name} - method: PUT - parameters: - cluster_name: test - request: - payload: - nodes_dn: - - CN=cluster3.example.com - response: - status: 400 - - synopsis: Get distinguished name. - path: /_plugins/_security/api/nodesdn/{cluster_name} - method: GET - parameters: - cluster_name: test - response: - status: 400 - - synopsis: Patch distinguished name. - path: /_plugins/_security/api/nodesdn/{cluster_name} - method: PATCH - parameters: - cluster_name: test - request: - payload: - op: replace - path: /test/nodes_dn/0 - value: [CN=cluster2.example.com] - response: - status: 400 - - synopsis: Delete distinguished name. - path: /_plugins/_security/api/nodesdn/{cluster_name} - method: DELETE - parameters: - cluster_name: test - response: - status: 400 diff --git a/tests/plugins/ml/ingest/pipeline/neural_search.yaml b/tests/plugins/ml/ingest/pipeline/neural_search.yaml index ca2fdd60a..5e68f5995 100644 --- a/tests/plugins/ml/ingest/pipeline/neural_search.yaml +++ b/tests/plugins/ml/ingest/pipeline/neural_search.yaml @@ -183,23 +183,6 @@ chapters: title: Moneyball script: source: _score * 1.7 - - synopsis: Search by Image - Invalid Model. - path: /{index}/_search - method: POST - parameters: - index: movies - request: - payload: - _source: - excludes: [passage_embedding] - query: - neural: - passage_embedding: - query_image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII= - model_id: invalid # This makes the test fail with a 404 but will still validate the query_image. - k: 100 - response: - status: 404 - synopsis: Undeploy a model. path: /_plugins/_ml/models/{model_id}/_undeploy method: POST diff --git a/tests/plugins/ml/ingest/pipeline/search_by_image.yaml b/tests/plugins/ml/ingest/pipeline/search_by_image.yaml new file mode 100644 index 000000000..e4fc46e61 --- /dev/null +++ b/tests/plugins/ml/ingest/pipeline/search_by_image.yaml @@ -0,0 +1,26 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test search by image. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +chapters: + - synopsis: Search by Image - Invalid Model. + path: /{index}/_search + method: POST + parameters: + index: movies + request: + payload: + _source: + excludes: [passage_embedding] + query: + neural: + passage_embedding: + query_image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII= + model_id: invalid # This makes the test fail with a 404 but will still validate the query_image. + k: 100 + response: + status: 404 diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index c463e49d9..9e8418688 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -50,8 +50,8 @@ export default class SchemasValidator { const message = this.json_validator.validate_schema(_schema) if (message == null) return - const file = `schemas/${key.split(':')[0]}.yaml` - const location = `#/components/schemas/${key.split(':')[1]}` + const file = `schemas/${key.split('___')[0]}.yaml` + const location = `#/components/schemas/${key.split('___')[1]}` return this.error(file, location, message) }).filter((error) => error != null) as ValidationError[] } @@ -61,9 +61,9 @@ export default class SchemasValidator { const message = this.json_validator.validate_schema(param.schema) if (message == null) return - const namespace = this.group_to_namespace(key.split('::')[0]) + const namespace = this.group_to_namespace(key.split('___')[0]) const file = namespace === '_global' ? '_global_parameters.yaml' : `namespaces/${namespace}.yaml` - const location = namespace === '_global' ? param.name as string : `#/components/parameters/${key}` + const location = namespace === '_global' ? param.name as string : `#/components/parameters/${key}`.replace('___', '::') return this.error(file, location, message) }).filter((error) => error != null) as ValidationError[] } @@ -78,9 +78,9 @@ export default class SchemasValidator { validate_response_schemas (): ValidationError[] { return Object.entries(this.spec.responses as Record).flatMap(([key, response]) => { - const namespace = this.group_to_namespace(key.split('@')[0]) + const namespace = this.group_to_namespace(key.split('___')[0]) const file = `namespaces/${namespace}.yaml` - const location = `#/components/responses/${key}` + const location = `#/components/responses/${key}`.replace('___', '@') const content = response.content as Record return this.validate_content_schemas(file, location, content) }) diff --git a/tools/src/merger/OpenApiMerger.ts b/tools/src/merger/OpenApiMerger.ts index ff8cd7d06..9ea66265d 100644 --- a/tools/src/merger/OpenApiMerger.ts +++ b/tools/src/merger/OpenApiMerger.ts @@ -53,8 +53,11 @@ export default class OpenApiMerger { this.#merge_schemas() this.#merge_namespaces() this.#sort_spec_keys() + this.#add_defaults() + this.#fix_refs() this.#generate_global_params() this.#generate_superseded_ops() + this.#normalize_fields() this._merged = true } @@ -134,6 +137,58 @@ export default class OpenApiMerger { }) } + #fix_refs(obj: any = this._spec.components): void { + if (obj?.$ref !== undefined) { + if (obj?.description !== undefined) { + delete obj?.description + } + } + + for (const key in obj) { + var item = obj[key] + if (_.isObject(item) || _.isArray(item)) { + this.#fix_refs(item) + } + } + } + + #normalize_key(key: string): string { + return key + .replaceAll('::', '___') + .replaceAll('@', '___') + .replaceAll(':', '___') + } + + #normalize_fields(obj: any = this._spec): void { + for (const key in obj) { + var item = obj[key] + + if (item?.$ref !== undefined) { + var renamed_ref = this.#normalize_key(item.$ref as string) + if (renamed_ref != item.$ref) { + item.$ref = renamed_ref + } + } + + var renamed_key = this.#normalize_key(key) + if (renamed_key != key) { + obj[renamed_key] = obj[key] + delete obj[key] + } + + if (_.isObject(item) || _.isArray(item)) { + this.#normalize_fields(item) + } + } + } + + #add_defaults(): void { + // Add default descriptions + Object.entries(this._spec.components.responses as Document).forEach(([_path, response_item]) => { + if (response_item.description === undefined) response_item.description = '' + }) + } + // Generate global parameters from _global_params.yaml file. #generate_global_params (): void { const gen = new GlobalParamsGenerator(this.root_folder) diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index f85e6fdc6..ad34d715b 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -61,9 +61,8 @@ export default class ChapterReader { } else { response.status = e.response.status response.content_type = e.response.headers['content-type']?.split(';')[0] - const payload = this.#deserialize_payload(e.response.data, response.content_type) - if (payload !== undefined) response.payload = payload.error ?? payload - response.message = payload.error?.reason ?? e.response.statusText + response.payload = this.#deserialize_payload(e.response.data, response.content_type) + response.message = response.payload.error?.reason ?? e.response.statusText this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } }) diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 6317ef392..77fe2df6d 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -19,6 +19,7 @@ import type StoryValidator from "./StoryValidator"; import { OpenSearchHttpClient } from 'OpenSearchHttpClient' import * as ansi from './Ansi' import _ from 'lodash' +import { Logger } from 'Logger' export default class TestRunner { private readonly _http_client: OpenSearchHttpClient @@ -26,12 +27,14 @@ export default class TestRunner { private readonly _story_evaluator: StoryEvaluator private readonly _result_logger: ResultLogger private readonly _story_files: Record = {} + private readonly _logger: Logger - constructor (http_client: OpenSearchHttpClient, story_validator: StoryValidator, story_evaluator: StoryEvaluator, result_logger: ResultLogger) { + constructor (http_client: OpenSearchHttpClient, story_validator: StoryValidator, story_evaluator: StoryEvaluator, result_logger: ResultLogger, logger: Logger) { this._http_client = http_client this._story_validator = story_validator this._story_evaluator = story_evaluator this._result_logger = result_logger + this._logger = logger } async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { @@ -52,6 +55,7 @@ export default class TestRunner { } for (const story_file of story_files) { + this._logger.info(`Evaluating ${story_file.display_path} ...`) const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index e781e343c..77351fe75 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -75,7 +75,7 @@ const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_ const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) -const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) +const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger, logger) runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( diff --git a/tools/src/validate-spec-py/Pipfile b/tools/src/validate-spec-py/Pipfile new file mode 100644 index 000000000..fa5bb05f6 --- /dev/null +++ b/tools/src/validate-spec-py/Pipfile @@ -0,0 +1,12 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +openapi_spec_validator = "*" + +[dev-packages] + +[requires] +python_version = "3" \ No newline at end of file diff --git a/tools/src/validate-spec-py/Pipfile.lock b/tools/src/validate-spec-py/Pipfile.lock new file mode 100644 index 000000000..cbb761748 --- /dev/null +++ b/tools/src/validate-spec-py/Pipfile.lock @@ -0,0 +1,455 @@ +{ + "_meta": { + "hash": { + "sha256": "8b96bf24637f13e9b0dd9a291f35f43962b4918dd76e52d5e7459c5ac620cdc4" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", + "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + ], + "markers": "python_version >= '3.7'", + "version": "==24.2.0" + }, + "certifi": { + "hashes": [ + "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", + "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + ], + "markers": "python_version >= '3.6'", + "version": "==2024.8.30" + }, + "charset-normalizer": { + "hashes": [ + "sha256:0099d79bdfcf5c1f0c2c72f91516702ebf8b0b8ddd8905f97a8aecf49712c621", + "sha256:0713f3adb9d03d49d365b70b84775d0a0d18e4ab08d12bc46baa6132ba78aaf6", + "sha256:07afec21bbbbf8a5cc3651aa96b980afe2526e7f048fdfb7f1014d84acc8b6d8", + "sha256:0b309d1747110feb25d7ed6b01afdec269c647d382c857ef4663bbe6ad95a912", + "sha256:0d99dd8ff461990f12d6e42c7347fd9ab2532fb70e9621ba520f9e8637161d7c", + "sha256:0de7b687289d3c1b3e8660d0741874abe7888100efe14bd0f9fd7141bcbda92b", + "sha256:1110e22af8ca26b90bd6364fe4c763329b0ebf1ee213ba32b68c73de5752323d", + "sha256:130272c698667a982a5d0e626851ceff662565379baf0ff2cc58067b81d4f11d", + "sha256:136815f06a3ae311fae551c3df1f998a1ebd01ddd424aa5603a4336997629e95", + "sha256:14215b71a762336254351b00ec720a8e85cada43b987da5a042e4ce3e82bd68e", + "sha256:1db4e7fefefd0f548d73e2e2e041f9df5c59e178b4c72fbac4cc6f535cfb1565", + "sha256:1ffd9493de4c922f2a38c2bf62b831dcec90ac673ed1ca182fe11b4d8e9f2a64", + "sha256:2006769bd1640bdf4d5641c69a3d63b71b81445473cac5ded39740a226fa88ab", + "sha256:20587d20f557fe189b7947d8e7ec5afa110ccf72a3128d61a2a387c3313f46be", + "sha256:223217c3d4f82c3ac5e29032b3f1c2eb0fb591b72161f86d93f5719079dae93e", + "sha256:27623ba66c183eca01bf9ff833875b459cad267aeeb044477fedac35e19ba907", + "sha256:285e96d9d53422efc0d7a17c60e59f37fbf3dfa942073f666db4ac71e8d726d0", + "sha256:2de62e8801ddfff069cd5c504ce3bc9672b23266597d4e4f50eda28846c322f2", + "sha256:2f6c34da58ea9c1a9515621f4d9ac379871a8f21168ba1b5e09d74250de5ad62", + "sha256:309a7de0a0ff3040acaebb35ec45d18db4b28232f21998851cfa709eeff49d62", + "sha256:35c404d74c2926d0287fbd63ed5d27eb911eb9e4a3bb2c6d294f3cfd4a9e0c23", + "sha256:3710a9751938947e6327ea9f3ea6332a09bf0ba0c09cae9cb1f250bd1f1549bc", + "sha256:3d59d125ffbd6d552765510e3f31ed75ebac2c7470c7274195b9161a32350284", + "sha256:40d3ff7fc90b98c637bda91c89d51264a3dcf210cade3a2c6f838c7268d7a4ca", + "sha256:425c5f215d0eecee9a56cdb703203dda90423247421bf0d67125add85d0c4455", + "sha256:43193c5cda5d612f247172016c4bb71251c784d7a4d9314677186a838ad34858", + "sha256:44aeb140295a2f0659e113b31cfe92c9061622cadbc9e2a2f7b8ef6b1e29ef4b", + "sha256:47334db71978b23ebcf3c0f9f5ee98b8d65992b65c9c4f2d34c2eaf5bcaf0594", + "sha256:4796efc4faf6b53a18e3d46343535caed491776a22af773f366534056c4e1fbc", + "sha256:4a51b48f42d9358460b78725283f04bddaf44a9358197b889657deba38f329db", + "sha256:4b67fdab07fdd3c10bb21edab3cbfe8cf5696f453afce75d815d9d7223fbe88b", + "sha256:4ec9dd88a5b71abfc74e9df5ebe7921c35cbb3b641181a531ca65cdb5e8e4dea", + "sha256:4f9fc98dad6c2eaa32fc3af1417d95b5e3d08aff968df0cd320066def971f9a6", + "sha256:54b6a92d009cbe2fb11054ba694bc9e284dad30a26757b1e372a1fdddaf21920", + "sha256:55f56e2ebd4e3bc50442fbc0888c9d8c94e4e06a933804e2af3e89e2f9c1c749", + "sha256:5726cf76c982532c1863fb64d8c6dd0e4c90b6ece9feb06c9f202417a31f7dd7", + "sha256:5d447056e2ca60382d460a604b6302d8db69476fd2015c81e7c35417cfabe4cd", + "sha256:5ed2e36c3e9b4f21dd9422f6893dec0abf2cca553af509b10cd630f878d3eb99", + "sha256:5ff2ed8194587faf56555927b3aa10e6fb69d931e33953943bc4f837dfee2242", + "sha256:62f60aebecfc7f4b82e3f639a7d1433a20ec32824db2199a11ad4f5e146ef5ee", + "sha256:63bc5c4ae26e4bc6be6469943b8253c0fd4e4186c43ad46e713ea61a0ba49129", + "sha256:6b40e8d38afe634559e398cc32b1472f376a4099c75fe6299ae607e404c033b2", + "sha256:6b493a043635eb376e50eedf7818f2f322eabbaa974e948bd8bdd29eb7ef2a51", + "sha256:6dba5d19c4dfab08e58d5b36304b3f92f3bd5d42c1a3fa37b5ba5cdf6dfcbcee", + "sha256:6fd30dc99682dc2c603c2b315bded2799019cea829f8bf57dc6b61efde6611c8", + "sha256:707b82d19e65c9bd28b81dde95249b07bf9f5b90ebe1ef17d9b57473f8a64b7b", + "sha256:7706f5850360ac01d80c89bcef1640683cc12ed87f42579dab6c5d3ed6888613", + "sha256:7782afc9b6b42200f7362858f9e73b1f8316afb276d316336c0ec3bd73312742", + "sha256:79983512b108e4a164b9c8d34de3992f76d48cadc9554c9e60b43f308988aabe", + "sha256:7f683ddc7eedd742e2889d2bfb96d69573fde1d92fcb811979cdb7165bb9c7d3", + "sha256:82357d85de703176b5587dbe6ade8ff67f9f69a41c0733cf2425378b49954de5", + "sha256:84450ba661fb96e9fd67629b93d2941c871ca86fc38d835d19d4225ff946a631", + "sha256:86f4e8cca779080f66ff4f191a685ced73d2f72d50216f7112185dc02b90b9b7", + "sha256:8cda06946eac330cbe6598f77bb54e690b4ca93f593dee1568ad22b04f347c15", + "sha256:8ce7fd6767a1cc5a92a639b391891bf1c268b03ec7e021c7d6d902285259685c", + "sha256:8ff4e7cdfdb1ab5698e675ca622e72d58a6fa2a8aa58195de0c0061288e6e3ea", + "sha256:9289fd5dddcf57bab41d044f1756550f9e7cf0c8e373b8cdf0ce8773dc4bd417", + "sha256:92a7e36b000bf022ef3dbb9c46bfe2d52c047d5e3f3343f43204263c5addc250", + "sha256:92db3c28b5b2a273346bebb24857fda45601aef6ae1c011c0a997106581e8a88", + "sha256:95c3c157765b031331dd4db3c775e58deaee050a3042fcad72cbc4189d7c8dca", + "sha256:980b4f289d1d90ca5efcf07958d3eb38ed9c0b7676bf2831a54d4f66f9c27dfa", + "sha256:9ae4ef0b3f6b41bad6366fb0ea4fc1d7ed051528e113a60fa2a65a9abb5b1d99", + "sha256:9c98230f5042f4945f957d006edccc2af1e03ed5e37ce7c373f00a5a4daa6149", + "sha256:9fa2566ca27d67c86569e8c85297aaf413ffab85a8960500f12ea34ff98e4c41", + "sha256:a14969b8691f7998e74663b77b4c36c0337cb1df552da83d5c9004a93afdb574", + "sha256:a8aacce6e2e1edcb6ac625fb0f8c3a9570ccc7bfba1f63419b3769ccf6a00ed0", + "sha256:a8e538f46104c815be19c975572d74afb53f29650ea2025bbfaef359d2de2f7f", + "sha256:aa41e526a5d4a9dfcfbab0716c7e8a1b215abd3f3df5a45cf18a12721d31cb5d", + "sha256:aa693779a8b50cd97570e5a0f343538a8dbd3e496fa5dcb87e29406ad0299654", + "sha256:ab22fbd9765e6954bc0bcff24c25ff71dcbfdb185fcdaca49e81bac68fe724d3", + "sha256:ab2e5bef076f5a235c3774b4f4028a680432cded7cad37bba0fd90d64b187d19", + "sha256:ab973df98fc99ab39080bfb0eb3a925181454d7c3ac8a1e695fddfae696d9e90", + "sha256:af73657b7a68211996527dbfeffbb0864e043d270580c5aef06dc4b659a4b578", + "sha256:b197e7094f232959f8f20541ead1d9862ac5ebea1d58e9849c1bf979255dfac9", + "sha256:b295729485b06c1a0683af02a9e42d2caa9db04a373dc38a6a58cdd1e8abddf1", + "sha256:b8831399554b92b72af5932cdbbd4ddc55c55f631bb13ff8fe4e6536a06c5c51", + "sha256:b8dcd239c743aa2f9c22ce674a145e0a25cb1566c495928440a181ca1ccf6719", + "sha256:bcb4f8ea87d03bc51ad04add8ceaf9b0f085ac045ab4d74e73bbc2dc033f0236", + "sha256:bd7af3717683bea4c87acd8c0d3d5b44d56120b26fd3f8a692bdd2d5260c620a", + "sha256:bf4475b82be41b07cc5e5ff94810e6a01f276e37c2d55571e3fe175e467a1a1c", + "sha256:c3e446d253bd88f6377260d07c895816ebf33ffffd56c1c792b13bff9c3e1ade", + "sha256:c57516e58fd17d03ebe67e181a4e4e2ccab1168f8c2976c6a334d4f819fe5944", + "sha256:c94057af19bc953643a33581844649a7fdab902624d2eb739738a30e2b3e60fc", + "sha256:cab5d0b79d987c67f3b9e9c53f54a61360422a5a0bc075f43cab5621d530c3b6", + "sha256:ce031db0408e487fd2775d745ce30a7cd2923667cf3b69d48d219f1d8f5ddeb6", + "sha256:cee4373f4d3ad28f1ab6290684d8e2ebdb9e7a1b74fdc39e4c211995f77bec27", + "sha256:d5b054862739d276e09928de37c79ddeec42a6e1bfc55863be96a36ba22926f6", + "sha256:dbe03226baf438ac4fda9e2d0715022fd579cb641c4cf639fa40d53b2fe6f3e2", + "sha256:dc15e99b2d8a656f8e666854404f1ba54765871104e50c8e9813af8a7db07f12", + "sha256:dcaf7c1524c0542ee2fc82cc8ec337f7a9f7edee2532421ab200d2b920fc97cf", + "sha256:dd4eda173a9fcccb5f2e2bd2a9f423d180194b1bf17cf59e3269899235b2a114", + "sha256:dd9a8bd8900e65504a305bf8ae6fa9fbc66de94178c420791d0293702fce2df7", + "sha256:de7376c29d95d6719048c194a9cf1a1b0393fbe8488a22008610b0361d834ecf", + "sha256:e7fdd52961feb4c96507aa649550ec2a0d527c086d284749b2f582f2d40a2e0d", + "sha256:e91f541a85298cf35433bf66f3fab2a4a2cff05c127eeca4af174f6d497f0d4b", + "sha256:e9e3c4c9e1ed40ea53acf11e2a386383c3304212c965773704e4603d589343ed", + "sha256:ee803480535c44e7f5ad00788526da7d85525cfefaf8acf8ab9a310000be4b03", + "sha256:f09cb5a7bbe1ecae6e87901a2eb23e0256bb524a79ccc53eb0b7629fbe7677c4", + "sha256:f19c1585933c82098c2a520f8ec1227f20e339e33aca8fa6f956f6691b784e67", + "sha256:f1a2f519ae173b5b6a2c9d5fa3116ce16e48b3462c8b96dfdded11055e3d6365", + "sha256:f28f891ccd15c514a0981f3b9db9aa23d62fe1a99997512b0491d2ed323d229a", + "sha256:f3e73a4255342d4eb26ef6df01e3962e73aa29baa3124a8e824c5d3364a65748", + "sha256:f606a1881d2663630ea5b8ce2efe2111740df4b687bd78b34a8131baa007f79b", + "sha256:fe9f97feb71aa9896b81973a7bbada8c49501dc73e58a10fcef6663af95e5079", + "sha256:ffc519621dce0c767e96b9c53f09c5d215578e10b02c285809f76509a3931482" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.4.0" + }, + "idna": { + "hashes": [ + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" + ], + "markers": "python_version >= '3.6'", + "version": "==3.10" + }, + "jsonschema": { + "hashes": [ + "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", + "sha256:fbadb6f8b144a8f8cf9f0b89ba94501d143e50411a1278633f56a7acf7fd5566" + ], + "markers": "python_version >= '3.8'", + "version": "==4.23.0" + }, + "jsonschema-path": { + "hashes": [ + "sha256:203aff257f8038cd3c67be614fe6b2001043408cb1b4e36576bc4921e09d83c4", + "sha256:f02e5481a4288ec062f8e68c808569e427d905bedfecb7f2e4c69ef77957c382" + ], + "markers": "python_full_version >= '3.8.0' and python_full_version < '4.0.0'", + "version": "==0.3.3" + }, + "jsonschema-specifications": { + "hashes": [ + "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc", + "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c" + ], + "markers": "python_version >= '3.8'", + "version": "==2023.12.1" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:009e6bb1f1935a62889ddc8541514b6a9e1fcf302667dcb049a0be5c8f613e56", + "sha256:02c83f957782cbbe8136bee26416686a6ae998c7b6191711a04da776dc9e47d4", + "sha256:0aefc7591920bbd360d57ea03c995cebc204b424524a5bd78406f6e1b8b2a5d8", + "sha256:127a789c75151db6af398b8972178afe6bda7d6f68730c057fbbc2e96b08d282", + "sha256:18dd842b49456aaa9a7cf535b04ca4571a302ff72ed8740d06b5adcd41fe0757", + "sha256:217138197c170a2a74ca0e05bddcd5f1796c735c37d0eee33e43259b192aa424", + "sha256:2297f08f08a2bb0d32a4265e98a006643cd7233fb7983032bd61ac7a02956b3b", + "sha256:2fc0a92c02fa1ca1e84fc60fa258458e5bf89d90a1ddaeb8ed9cc3147f417255", + "sha256:30b339b2a743c5288405aa79a69e706a06e02958eab31859f7f3c04980853b70", + "sha256:366c32fe5355ef5fc8a232c5436f4cc66e9d3e8967c01fb2e6302fd6627e3d94", + "sha256:3ad54b9ddbe20ae9f7c1b29e52f123120772b06dbb18ec6be9101369d63a4074", + "sha256:5ad9e6ed739285919aa9661a5bbed0aaf410aa60231373c5579c6b4801bd883c", + "sha256:5faf03a7d8942bb4476e3b62fd0f4cf94eaf4618e304a19865abf89a35c0bbee", + "sha256:75fc59fc450050b1b3c203c35020bc41bd2695ed692a392924c6ce180c6f1dc9", + "sha256:76a095cfe6045c7d0ca77db9934e8f7b71b14645f0094ffcd842349ada5c5fb9", + "sha256:78247b6d45f43a52ef35c25b5581459e85117225408a4128a3daf8bf9648ac69", + "sha256:782e2c9b2aab1708ffb07d4bf377d12901d7a1d99e5e410d648d892f8967ab1f", + "sha256:7ab7004cf2e59f7c2e4345604a3e6ea0d92ac44e1c2375527d56492014e690c3", + "sha256:80b39d3a151309efc8cc48675918891b865bdf742a8616a337cb0090791a0de9", + "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d", + "sha256:855e068b0358ab916454464a884779c7ffa312b8925c6f7401e952dcf3b89977", + "sha256:92f09ff65ecff3108e56526f9e2481b8116c0b9e1425325e13245abfd79bdb1b", + "sha256:952c81d415b9b80ea261d2372d2a4a2332a3890c2b83e0535f263ddfe43f0d43", + "sha256:9a3a87cf1e133e5b1994144c12ca4aa3d9698517fe1e2ca82977781b16955658", + "sha256:9e4ed0518a14dd26092614412936920ad081a424bdcb54cc13349a8e2c6d106a", + "sha256:a899b10e17743683b293a729d3a11f2f399e8a90c73b089e29f5d0fe3509f0dd", + "sha256:b1f711e2c6dcd4edd372cf5dec5c5a30d23bba06ee012093267b3376c079ec83", + "sha256:b4f87d4ed9064b2628da63830986c3d2dca7501e6018347798313fcf028e2fd4", + "sha256:cb73507defd385b7705c599a94474b1d5222a508e502553ef94114a143ec6696", + "sha256:dc0d2fc424e54c70c4bc06787e4072c4f3b1aa2f897dfdc34ce1013cf3ceef05", + "sha256:e221060b701e2aa2ea991542900dd13907a5c90fa80e199dbf5a03359019e7a3", + "sha256:e271058822765ad5e3bca7f05f2ace0de58a3f4e62045a8c90a0dfd2f8ad8cc6", + "sha256:e2adb09778797da09d2b5ebdbceebf7dd32e2c96f79da9052b2e87b6ea495895", + "sha256:e333e2324307a7b5d86adfa835bb500ee70bfcd1447384a822e96495796b0ca4", + "sha256:e98c8af98d5707dcdecc9ab0863c0ea6e88545d42ca7c3feffb6b4d1e370c7ba", + "sha256:edb45bb8278574710e68a6b021599a10ce730d156e5b254941754a9cc0b17d03", + "sha256:fec03caabbc6b59ea4a638bee5fce7117be8e99a4103d9d5ad77f15d6f81020c" + ], + "markers": "python_version >= '3.8'", + "version": "==1.10.0" + }, + "openapi-schema-validator": { + "hashes": [ + "sha256:11a95c9c9017912964e3e5f2545a5b11c3814880681fcacfb73b1759bb4f2804", + "sha256:c4887c1347c669eb7cded9090f4438b710845cd0f90d1fb9e1b3303fb37339f8" + ], + "markers": "python_full_version >= '3.8.0' and python_full_version < '4.0.0'", + "version": "==0.6.2" + }, + "openapi-spec-validator": { + "hashes": [ + "sha256:3c81825043f24ccbcd2f4b149b11e8231abce5ba84f37065e14ec947d8f4e959", + "sha256:8577b85a8268685da6f8aa30990b83b7960d4d1117e901d451b5d572605e5ec7" + ], + "markers": "python_full_version >= '3.8.0' and python_full_version < '4.0.0'", + "version": "==0.7.1" + }, + "pathable": { + "hashes": [ + "sha256:5c869d315be50776cc8a993f3af43e0c60dc01506b399643f919034ebf4cdcab", + "sha256:cdd7b1f9d7d5c8b8d3315dbf5a86b2596053ae845f056f57d97c0eefff84da14" + ], + "markers": "python_full_version >= '3.7.0' and python_full_version < '4.0.0'", + "version": "==0.4.3" + }, + "pyyaml": { + "hashes": [ + "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", + "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", + "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", + "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", + "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", + "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", + "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", + "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", + "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", + "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", + "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", + "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", + "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", + "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", + "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", + "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", + "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", + "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", + "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", + "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", + "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", + "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", + "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", + "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", + "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", + "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", + "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", + "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", + "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", + "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", + "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", + "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", + "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", + "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", + "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", + "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", + "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", + "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", + "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", + "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", + "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", + "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", + "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", + "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", + "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", + "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", + "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", + "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", + "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", + "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", + "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", + "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", + "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4" + ], + "markers": "python_version >= '3.8'", + "version": "==6.0.2" + }, + "referencing": { + "hashes": [ + "sha256:25b42124a6c8b632a425174f24087783efb348a6f1e0008e63cd4466fedf703c", + "sha256:eda6d3234d62814d1c64e305c1331c9a3a6132da475ab6382eaa997b21ee75de" + ], + "markers": "python_version >= '3.8'", + "version": "==0.35.1" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" + }, + "rfc3339-validator": { + "hashes": [ + "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", + "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.1.4" + }, + "rpds-py": { + "hashes": [ + "sha256:06db23d43f26478303e954c34c75182356ca9aa7797d22c5345b16871ab9c45c", + "sha256:0e13e6952ef264c40587d510ad676a988df19adea20444c2b295e536457bc585", + "sha256:11ef6ce74616342888b69878d45e9f779b95d4bd48b382a229fe624a409b72c5", + "sha256:1259c7b3705ac0a0bd38197565a5d603218591d3f6cee6e614e380b6ba61c6f6", + "sha256:18d7585c463087bddcfa74c2ba267339f14f2515158ac4db30b1f9cbdb62c8ef", + "sha256:1e0f80b739e5a8f54837be5d5c924483996b603d5502bfff79bf33da06164ee2", + "sha256:1e5f3cd7397c8f86c8cc72d5a791071431c108edd79872cdd96e00abd8497d29", + "sha256:220002c1b846db9afd83371d08d239fdc865e8f8c5795bbaec20916a76db3318", + "sha256:22e6c9976e38f4d8c4a63bd8a8edac5307dffd3ee7e6026d97f3cc3a2dc02a0b", + "sha256:238a2d5b1cad28cdc6ed15faf93a998336eb041c4e440dd7f902528b8891b399", + "sha256:2580b0c34583b85efec8c5c5ec9edf2dfe817330cc882ee972ae650e7b5ef739", + "sha256:28527c685f237c05445efec62426d285e47a58fb05ba0090a4340b73ecda6dee", + "sha256:2cf126d33a91ee6eedc7f3197b53e87a2acdac63602c0f03a02dd69e4b138174", + "sha256:338ca4539aad4ce70a656e5187a3a31c5204f261aef9f6ab50e50bcdffaf050a", + "sha256:39ed0d010457a78f54090fafb5d108501b5aa5604cc22408fc1c0c77eac14344", + "sha256:3ad0fda1635f8439cde85c700f964b23ed5fc2d28016b32b9ee5fe30da5c84e2", + "sha256:3d2b1ad682a3dfda2a4e8ad8572f3100f95fad98cb99faf37ff0ddfe9cbf9d03", + "sha256:3d61339e9f84a3f0767b1995adfb171a0d00a1185192718a17af6e124728e0f5", + "sha256:3fde368e9140312b6e8b6c09fb9f8c8c2f00999d1823403ae90cc00480221b22", + "sha256:40ce74fc86ee4645d0a225498d091d8bc61f39b709ebef8204cb8b5a464d3c0e", + "sha256:49a8063ea4296b3a7e81a5dfb8f7b2d73f0b1c20c2af401fb0cdf22e14711a96", + "sha256:4a1f1d51eccb7e6c32ae89243cb352389228ea62f89cd80823ea7dd1b98e0b91", + "sha256:4b16aa0107ecb512b568244ef461f27697164d9a68d8b35090e9b0c1c8b27752", + "sha256:4f1ed4749a08379555cebf4650453f14452eaa9c43d0a95c49db50c18b7da075", + "sha256:4fe84294c7019456e56d93e8ababdad5a329cd25975be749c3f5f558abb48253", + "sha256:50eccbf054e62a7b2209b28dc7a22d6254860209d6753e6b78cfaeb0075d7bee", + "sha256:514b3293b64187172bc77c8fb0cdae26981618021053b30d8371c3a902d4d5ad", + "sha256:54b43a2b07db18314669092bb2de584524d1ef414588780261e31e85846c26a5", + "sha256:55fea87029cded5df854ca7e192ec7bdb7ecd1d9a3f63d5c4eb09148acf4a7ce", + "sha256:569b3ea770c2717b730b61998b6c54996adee3cef69fc28d444f3e7920313cf7", + "sha256:56e27147a5a4c2c21633ff8475d185734c0e4befd1c989b5b95a5d0db699b21b", + "sha256:57eb94a8c16ab08fef6404301c38318e2c5a32216bf5de453e2714c964c125c8", + "sha256:5a35df9f5548fd79cb2f52d27182108c3e6641a4feb0f39067911bf2adaa3e57", + "sha256:5a8c94dad2e45324fc74dce25e1645d4d14df9a4e54a30fa0ae8bad9a63928e3", + "sha256:5b4f105deeffa28bbcdff6c49b34e74903139afa690e35d2d9e3c2c2fba18cec", + "sha256:5c1dc0f53856b9cc9a0ccca0a7cc61d3d20a7088201c0937f3f4048c1718a209", + "sha256:614fdafe9f5f19c63ea02817fa4861c606a59a604a77c8cdef5aa01d28b97921", + "sha256:617c7357272c67696fd052811e352ac54ed1d9b49ab370261a80d3b6ce385045", + "sha256:65794e4048ee837494aea3c21a28ad5fc080994dfba5b036cf84de37f7ad5074", + "sha256:6632f2d04f15d1bd6fe0eedd3b86d9061b836ddca4c03d5cf5c7e9e6b7c14580", + "sha256:6c8ef2ebf76df43f5750b46851ed1cdf8f109d7787ca40035fe19fbdc1acc5a7", + "sha256:758406267907b3781beee0f0edfe4a179fbd97c0be2e9b1154d7f0a1279cf8e5", + "sha256:7e60cb630f674a31f0368ed32b2a6b4331b8350d67de53c0359992444b116dd3", + "sha256:89c19a494bf3ad08c1da49445cc5d13d8fefc265f48ee7e7556839acdacf69d0", + "sha256:8a86a9b96070674fc88b6f9f71a97d2c1d3e5165574615d1f9168ecba4cecb24", + "sha256:8bc7690f7caee50b04a79bf017a8d020c1f48c2a1077ffe172abec59870f1139", + "sha256:8d7919548df3f25374a1f5d01fbcd38dacab338ef5f33e044744b5c36729c8db", + "sha256:9426133526f69fcaba6e42146b4e12d6bc6c839b8b555097020e2b78ce908dcc", + "sha256:9824fb430c9cf9af743cf7aaf6707bf14323fb51ee74425c380f4c846ea70789", + "sha256:9bb4a0d90fdb03437c109a17eade42dfbf6190408f29b2744114d11586611d6f", + "sha256:9bc2d153989e3216b0559251b0c260cfd168ec78b1fac33dd485750a228db5a2", + "sha256:9d35cef91e59ebbeaa45214861874bc6f19eb35de96db73e467a8358d701a96c", + "sha256:a1862d2d7ce1674cffa6d186d53ca95c6e17ed2b06b3f4c476173565c862d232", + "sha256:a84ab91cbe7aab97f7446652d0ed37d35b68a465aeef8fc41932a9d7eee2c1a6", + "sha256:aa7f429242aae2947246587d2964fad750b79e8c233a2367f71b554e9447949c", + "sha256:aa9a0521aeca7d4941499a73ad7d4f8ffa3d1affc50b9ea11d992cd7eff18a29", + "sha256:ac2f4f7a98934c2ed6505aead07b979e6f999389f16b714448fb39bbaa86a489", + "sha256:ae94bd0b2f02c28e199e9bc51485d0c5601f58780636185660f86bf80c89af94", + "sha256:af0fc424a5842a11e28956e69395fbbeab2c97c42253169d87e90aac2886d751", + "sha256:b2a5db5397d82fa847e4c624b0c98fe59d2d9b7cf0ce6de09e4d2e80f8f5b3f2", + "sha256:b4c29cbbba378759ac5786730d1c3cb4ec6f8ababf5c42a9ce303dc4b3d08cda", + "sha256:b74b25f024b421d5859d156750ea9a65651793d51b76a2e9238c05c9d5f203a9", + "sha256:b7f19250ceef892adf27f0399b9e5afad019288e9be756d6919cb58892129f51", + "sha256:b80d4a7900cf6b66bb9cee5c352b2d708e29e5a37fe9bf784fa97fc11504bf6c", + "sha256:b8c00a3b1e70c1d3891f0db1b05292747f0dbcfb49c43f9244d04c70fbc40eb8", + "sha256:bb273176be34a746bdac0b0d7e4e2c467323d13640b736c4c477881a3220a989", + "sha256:c3c20f0ddeb6e29126d45f89206b8291352b8c5b44384e78a6499d68b52ae511", + "sha256:c3e130fd0ec56cb76eb49ef52faead8ff09d13f4527e9b0c400307ff72b408e1", + "sha256:c52d3f2f82b763a24ef52f5d24358553e8403ce05f893b5347098014f2d9eff2", + "sha256:c6377e647bbfd0a0b159fe557f2c6c602c159fc752fa316572f012fc0bf67150", + "sha256:c638144ce971df84650d3ed0096e2ae7af8e62ecbbb7b201c8935c370df00a2c", + "sha256:ce9845054c13696f7af7f2b353e6b4f676dab1b4b215d7fe5e05c6f8bb06f965", + "sha256:cf258ede5bc22a45c8e726b29835b9303c285ab46fc7c3a4cc770736b5304c9f", + "sha256:d0a26ffe9d4dd35e4dfdd1e71f46401cff0181c75ac174711ccff0459135fa58", + "sha256:d0b67d87bb45ed1cd020e8fbf2307d449b68abc45402fe1a4ac9e46c3c8b192b", + "sha256:d20277fd62e1b992a50c43f13fbe13277a31f8c9f70d59759c88f644d66c619f", + "sha256:d454b8749b4bd70dd0a79f428731ee263fa6995f83ccb8bada706e8d1d3ff89d", + "sha256:d4c7d1a051eeb39f5c9547e82ea27cbcc28338482242e3e0b7768033cb083821", + "sha256:d72278a30111e5b5525c1dd96120d9e958464316f55adb030433ea905866f4de", + "sha256:d72a210824facfdaf8768cf2d7ca25a042c30320b3020de2fa04640920d4e121", + "sha256:d807dc2051abe041b6649681dce568f8e10668e3c1c6543ebae58f2d7e617855", + "sha256:dbe982f38565bb50cb7fb061ebf762c2f254ca3d8c20d4006878766e84266272", + "sha256:dcedf0b42bcb4cfff4101d7771a10532415a6106062f005ab97d1d0ab5681c60", + "sha256:deb62214c42a261cb3eb04d474f7155279c1a8a8c30ac89b7dcb1721d92c3c02", + "sha256:def7400461c3a3f26e49078302e1c1b38f6752342c77e3cf72ce91ca69fb1bc1", + "sha256:df3de6b7726b52966edf29663e57306b23ef775faf0ac01a3e9f4012a24a4140", + "sha256:e1940dae14e715e2e02dfd5b0f64a52e8374a517a1e531ad9412319dc3ac7879", + "sha256:e4df1e3b3bec320790f699890d41c59d250f6beda159ea3c44c3f5bac1976940", + "sha256:e6900ecdd50ce0facf703f7a00df12374b74bbc8ad9fe0f6559947fb20f82364", + "sha256:ea438162a9fcbee3ecf36c23e6c68237479f89f962f82dae83dc15feeceb37e4", + "sha256:eb851b7df9dda52dc1415ebee12362047ce771fc36914586b2e9fcbd7d293b3e", + "sha256:ec31a99ca63bf3cd7f1a5ac9fe95c5e2d060d3c768a09bc1d16e235840861420", + "sha256:f0475242f447cc6cb8a9dd486d68b2ef7fbee84427124c232bff5f63b1fe11e5", + "sha256:f2fbf7db2012d4876fb0d66b5b9ba6591197b0f165db8d99371d976546472a24", + "sha256:f60012a73aa396be721558caa3a6fd49b3dd0033d1675c6d59c4502e870fcf0c", + "sha256:f8e604fe73ba048c06085beaf51147eaec7df856824bfe7b98657cf436623daf", + "sha256:f90a4cd061914a60bd51c68bcb4357086991bd0bb93d8aa66a6da7701370708f", + "sha256:f918a1a130a6dfe1d7fe0f105064141342e7dd1611f2e6a21cd2f5c8cb1cfb3e", + "sha256:fa518bcd7600c584bf42e6617ee8132869e877db2f76bcdc281ec6a4113a53ab", + "sha256:faefcc78f53a88f3076b7f8be0a8f8d35133a3ecf7f3770895c25f8813460f08", + "sha256:fcaeb7b57f1a1e071ebd748984359fef83ecb026325b9d4ca847c95bc7311c92", + "sha256:fd2d84f40633bc475ef2d5490b9c19543fbf18596dcb1b291e3a12ea5d722f7a", + "sha256:fdfc3a892927458d98f3d55428ae46b921d1f7543b89382fdb483f5640daaec8" + ], + "markers": "python_version >= '3.8'", + "version": "==0.20.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", + "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9" + ], + "markers": "python_version >= '3.8'", + "version": "==2.2.3" + } + }, + "develop": {} +} diff --git a/tools/src/validate-spec-py/validate.py b/tools/src/validate-spec-py/validate.py new file mode 100644 index 000000000..9739c51f6 --- /dev/null +++ b/tools/src/validate-spec-py/validate.py @@ -0,0 +1,20 @@ +import sys + +from openapi_spec_validator import validate +from openapi_spec_validator.readers import read_from_filename +from openapi_spec_validator.validation.exceptions import OpenAPIValidationError + +if len(sys.argv) < 2: + print("syntax: validate.py [spec]") + exit(1) + +spec = sys.argv[1] +print(f'Validating {spec} ...') + +spec_dict, base_uri = read_from_filename(spec) + +try: + validate(spec_dict) +except OpenAPIValidationError as err: + print(err) + exit(2) diff --git a/tools/src/validate-spec-ruby/Gemfile b/tools/src/validate-spec-ruby/Gemfile new file mode 100644 index 000000000..a7a8f3b53 --- /dev/null +++ b/tools/src/validate-spec-ruby/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' + +gem 'json_schemer' diff --git a/tools/src/validate-spec-ruby/Gemfile.lock b/tools/src/validate-spec-ruby/Gemfile.lock new file mode 100644 index 000000000..f6926e79f --- /dev/null +++ b/tools/src/validate-spec-ruby/Gemfile.lock @@ -0,0 +1,22 @@ +GEM + remote: https://rubygems.org/ + specs: + bigdecimal (3.1.8) + hana (1.3.7) + json_schemer (2.3.0) + bigdecimal + hana (~> 1.3) + regexp_parser (~> 2.0) + simpleidn (~> 0.2) + regexp_parser (2.9.2) + simpleidn (0.2.3) + +PLATFORMS + arm64-darwin-21 + ruby + +DEPENDENCIES + json_schemer + +BUNDLED WITH + 2.5.2 diff --git a/tools/src/validate-spec-ruby/validate.rb b/tools/src/validate-spec-ruby/validate.rb new file mode 100644 index 000000000..261cfd360 --- /dev/null +++ b/tools/src/validate-spec-ruby/validate.rb @@ -0,0 +1,20 @@ +require 'json_schemer' +require 'yaml' + +raise 'syntax: validate [spec]' unless ARGV.length >= 1 + +spec = ARGV[0] +puts "Validating #{spec} ..." + +schemer = JSONSchemer.openapi(YAML.load_file(spec)) +schemer.validate + +total_errors = 0 +schemer.validate.each do |error| + puts "#{error['data']}: #{error['error']}" if total_errors < 10 + total_errors += 1 +end + +puts total_errors > 0 ? " .... #{total_errors} total." : "Done." + +exit total_errors == 0 ? 0 : 1 diff --git a/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml index 24d08aa4d..eafc740f0 100644 --- a/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/opensearch/expected_1.3.yaml @@ -12,33 +12,33 @@ paths: x-version-added: '1.0' description: Returns information about nodes in the cluster. parameters: - - $ref: '#/components/parameters/nodes.info::path.id' + - $ref: '#/components/parameters/nodes.info___path.id' responses: '200': - $ref: '#/components/responses/nodes.info@200' + $ref: '#/components/responses/nodes.info___200' /index: get: operationId: get.0 responses: '200': - $ref: '#/components/responses/info@200' + $ref: '#/components/responses/info___200' '201': - $ref: '#/components/responses/info@201' + $ref: '#/components/responses/info___201' '404': - $ref: '#/components/responses/info@404' + $ref: '#/components/responses/info___404' '500': - $ref: '#/components/responses/info@500' + $ref: '#/components/responses/info___500' '503': - $ref: '#/components/responses/info@503' + $ref: '#/components/responses/info___503' removed-2.0: - $ref: '#/components/responses/info@removed-2.0' + $ref: '#/components/responses/info___removed-2.0' x-version-removed: '2.0' removed-2.0-refs: - $ref: '#/components/responses/info@removed-2.0-refs' + $ref: '#/components/responses/info___removed-2.0-refs' added-1.3-removed-2.0: - $ref: '#/components/responses/info@added-1.3-removed-2.0' + $ref: '#/components/responses/info___added-1.3-removed-2.0' distributed-excluded-amazon-serverless: - $ref: '#/components/responses/info@distributed-all' + $ref: '#/components/responses/info___distributed-all' x-distributions-excluded: - amazon-serverless parameters: [] @@ -47,11 +47,11 @@ paths: operationId: nodes.0 responses: '200': - $ref: '#/components/responses/nodes.info@200' + $ref: '#/components/responses/nodes.info___200' parameters: [] components: parameters: - nodes.info::path.id: + nodes.info___path.id: in: path name: id description: Node ID. @@ -60,19 +60,20 @@ components: type: string requestBodies: {} responses: - info@200: + info___200: content: application/json: schema: type: object properties: _type: - $ref: '#/components/schemas/_common:Type' + $ref: '#/components/schemas/_common___Type' tagline: type: string required: - tagline - info@201: + description: '' + info___201: content: application/json: schema: @@ -83,7 +84,8 @@ components: required: - tagline unevaluatedProperties: true - info@404: + description: '' + info___404: content: application/json: schema: @@ -95,7 +97,8 @@ components: - tagline unevaluatedProperties: type: object - info@500: + description: '' + info___500: content: application/json: schema: @@ -103,34 +106,36 @@ components: properties: tagline: type: string - info@503: + description: '' + info___503: content: application/json: schema: type: object - info@added-1.3-removed-2.0: + description: '' + info___added-1.3-removed-2.0: description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' - info@distributed-all: + info___distributed-all: description: Distributed in opensearch.org, AOS and AOSS. - info@removed-2.0: + info___removed-2.0: description: Removed in 2.0 via attribute next to ref. - info@removed-2.0-refs: + info___removed-2.0-refs: description: One of the ref values removed in 2.0. schema: oneOf: - - $ref: '#/components/schemas/_common:Type' - - $ref: '#/components/schemas/_common:OldId' - nodes.info@200: + - $ref: '#/components/schemas/_common___Type' + - $ref: '#/components/schemas/_common___OldId' + nodes.info___200: description: All nodes. content: application/json: schema: type: object schemas: - _common:OldId: + _common___OldId: type: string - _common:Type: + _common___Type: type: string x-version-removed: '2.0' diff --git a/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml index a824f4787..b287d6982 100644 --- a/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/opensearch/expected_2.0.yaml @@ -12,25 +12,25 @@ paths: x-version-added: '1.0' description: Returns information about nodes in the cluster. parameters: - - $ref: '#/components/parameters/nodes.info::path.id' + - $ref: '#/components/parameters/nodes.info___path.id' responses: '200': - $ref: '#/components/responses/nodes.info@200' + $ref: '#/components/responses/nodes.info___200' post: operationId: nodes.info.1 x-operation-group: nodes.info x-version-added: '2.0' description: Returns information about nodes in the cluster. parameters: - - $ref: '#/components/parameters/nodes.info::path.id' - - $ref: '#/components/parameters/nodes.info::query.flag' + - $ref: '#/components/parameters/nodes.info___path.id' + - $ref: '#/components/parameters/nodes.info___query.flag' requestBody: $ref: '#/components/requestBodies/nodes.info' responses: '200': - $ref: '#/components/responses/nodes.info@200' + $ref: '#/components/responses/nodes.info___200' '201': - $ref: '#/components/responses/nodes.info@201' + $ref: '#/components/responses/nodes.info___201' /cluster_manager: get: operationId: cluster_manager.0 @@ -45,22 +45,22 @@ paths: operationId: get.0 responses: '200': - $ref: '#/components/responses/info@200' + $ref: '#/components/responses/info___200' '201': - $ref: '#/components/responses/info@201' + $ref: '#/components/responses/info___201' '404': - $ref: '#/components/responses/info@404' + $ref: '#/components/responses/info___404' '500': - $ref: '#/components/responses/info@500' + $ref: '#/components/responses/info___500' '503': - $ref: '#/components/responses/info@503' + $ref: '#/components/responses/info___503' added-2.0: - $ref: '#/components/responses/info@added-2.0' + $ref: '#/components/responses/info___added-2.0' x-version-added: '2.0' removed-2.0-refs: - $ref: '#/components/responses/info@removed-2.0-refs' + $ref: '#/components/responses/info___removed-2.0-refs' distributed-excluded-amazon-serverless: - $ref: '#/components/responses/info@distributed-all' + $ref: '#/components/responses/info___distributed-all' x-distributions-excluded: - amazon-serverless parameters: [] @@ -69,18 +69,18 @@ paths: operationId: nodes.0 responses: '200': - $ref: '#/components/responses/nodes.info@200' + $ref: '#/components/responses/nodes.info___200' parameters: [] components: parameters: - nodes.info::path.id: + nodes.info___path.id: in: path name: id description: Node ID. required: true schema: type: string - nodes.info::query.flag: + nodes.info___query.flag: in: query name: flag description: Flag. @@ -98,11 +98,11 @@ components: _all: type: boolean ids: - $ref: '#/components/schemas/_common:Ids' + $ref: '#/components/schemas/_common___Ids' x-version-added: '2.0' description: Nodes options. responses: - info@200: + info___200: content: application/json: schema: @@ -112,7 +112,8 @@ components: type: string required: - tagline - info@201: + description: '' + info___201: content: application/json: schema: @@ -123,7 +124,8 @@ components: required: - tagline unevaluatedProperties: true - info@404: + description: '' + info___404: content: application/json: schema: @@ -135,7 +137,8 @@ components: - tagline unevaluatedProperties: type: object - info@500: + description: '' + info___500: content: application/json: schema: @@ -143,38 +146,40 @@ components: properties: tagline: type: string - info@503: + description: '' + info___503: content: application/json: schema: type: object - info@added-2.0: + description: '' + info___added-2.0: description: Added in 2.0 via attribute next to ref. - info@distributed-all: + info___distributed-all: description: Distributed in opensearch.org, AOS and AOSS. - info@removed-2.0-refs: + info___removed-2.0-refs: description: One of the ref values removed in 2.0. schema: oneOf: - - $ref: '#/components/schemas/_common:OldId' - nodes.info@200: + - $ref: '#/components/schemas/_common___OldId' + nodes.info___200: description: All nodes. content: application/json: schema: type: object - nodes.info@201: + nodes.info___201: description: All nodes. content: application/json: schema: type: object schemas: - _common:Ids: + _common___Ids: oneOf: - - $ref: '#/components/schemas/_common:OldId' + - $ref: '#/components/schemas/_common___OldId' - type: array items: - $ref: '#/components/schemas/_common:Ids' - _common:OldId: + $ref: '#/components/schemas/_common___Ids' + _common___OldId: type: string diff --git a/tools/tests/merger/fixtures/merger/animals/expected.yaml b/tools/tests/merger/fixtures/merger/animals/expected.yaml index 789d1f203..a28b20af2 100644 --- a/tools/tests/merger/fixtures/merger/animals/expected.yaml +++ b/tools/tests/merger/fixtures/merger/animals/expected.yaml @@ -7,54 +7,54 @@ paths: /{index}: post: parameters: - - $ref: '#/components/parameters/indices.create::path.index' - - $ref: '#/components/parameters/indices.create::query.pretty' - - $ref: '#/components/parameters/indices.create::query.wait_for_active_shards' - - $ref: '#/components/parameters/_global::query.human' + - $ref: '#/components/parameters/indices.create___path.index' + - $ref: '#/components/parameters/indices.create___query.pretty' + - $ref: '#/components/parameters/indices.create___query.wait_for_active_shards' + - $ref: '#/components/parameters/_global___query.human' requestBody: $ref: '#/components/requestBodies/indices.create' responses: '200': - $ref: '#/components/responses/indices.create@200' + $ref: '#/components/responses/indices.create___200' '201': - $ref: '#/components/responses/indices.create@201' + $ref: '#/components/responses/indices.create___201' x-version-added: '2.0' /adopt/{animal}/dockets/{docket}: get: operationId: adopt.0 parameters: - - $ref: '#/components/parameters/adopt::path.animal' - - $ref: '#/components/parameters/adopt::path.docket' - - $ref: '#/components/parameters/_global::query.human' + - $ref: '#/components/parameters/adopt___path.animal' + - $ref: '#/components/parameters/adopt___path.docket' + - $ref: '#/components/parameters/_global___query.human' responses: '200': - $ref: '#/components/responses/adopt@200' + $ref: '#/components/responses/adopt___200' post: operationId: adopt.1 parameters: - - $ref: '#/components/parameters/adopt::path.animal' - - $ref: '#/components/parameters/adopt::path.docket' - - $ref: '#/components/parameters/_global::query.human' + - $ref: '#/components/parameters/adopt___path.animal' + - $ref: '#/components/parameters/adopt___path.docket' + - $ref: '#/components/parameters/_global___query.human' requestBody: $ref: '#/components/requestBodies/adopt' responses: '200': - $ref: '#/components/responses/adopt@200' + $ref: '#/components/responses/adopt___200' /replaced/adopting/{animal}/something/{docket}: get: operationId: adopt.0_superseded parameters: - - $ref: '#/components/parameters/adopt::path.animal' - - $ref: '#/components/parameters/adopt::path.docket' - - $ref: '#/components/parameters/_global::query.human' + - $ref: '#/components/parameters/adopt___path.animal' + - $ref: '#/components/parameters/adopt___path.docket' + - $ref: '#/components/parameters/_global___query.human' responses: '200': - $ref: '#/components/responses/adopt@200' + $ref: '#/components/responses/adopt___200' deprecated: true x-ignorable: true components: parameters: - _global::query.human: + _global___query.human: name: human in: query description: Whether to return human readable values for statistics. @@ -62,27 +62,27 @@ components: type: boolean default: true x-global: true - adopt::path.animal: + adopt___path.animal: name: animal in: path schema: - $ref: '#/components/schemas/animals:Animal' - adopt::path.docket: + $ref: '#/components/schemas/animals___Animal' + adopt___path.docket: name: docket in: path schema: type: number - indices.create::path.index: + indices.create___path.index: name: index in: path schema: type: string - indices.create::query.pretty: + indices.create___query.pretty: name: pretty in: query schema: type: boolean - indices.create::query.wait_for_active_shards: + indices.create___query.wait_for_active_shards: name: pretty in: query x-version-added: '2.0' @@ -94,35 +94,37 @@ components: name: type: string responses: - adopt@200: + adopt___200: application/json: schema: type: object - indices.create@200: + description: '' + indices.create___200: application/json: schema: type: object - indices.create@201: + description: '' + indices.create___201: description: Added in 2.0. application/json: schema: type: object schemas: - actions:Bark: + actions___Bark: type: string - actions:Meow: + actions___Meow: type: string - animals:Animal: + animals___Animal: oneOf: - - $ref: '#/components/schemas/animals:Dog' - - $ref: '#/components/schemas/animals:Cat' - animals:Cat: + - $ref: '#/components/schemas/animals___Dog' + - $ref: '#/components/schemas/animals___Cat' + animals___Cat: type: object properties: meow: - $ref: '#/components/schemas/actions:Meow' - animals:Dog: + $ref: '#/components/schemas/actions___Meow' + animals___Dog: type: object properties: bark: - $ref: '#/components/schemas/actions:Bark' + $ref: '#/components/schemas/actions___Bark' diff --git a/tools/tests/tester/ChapterReader.test.ts b/tools/tests/tester/ChapterReader.test.ts index 38a0b82fc..2b0c2eb42 100644 --- a/tools/tests/tester/ChapterReader.test.ts +++ b/tools/tests/tester/ChapterReader.test.ts @@ -248,6 +248,35 @@ describe('ChapterReader', () => { message: 'Not Found' }); }); + + it('sets payload to entire response when payload.error is present', async () => { + const mock_payload = { '_data': '1', 'result': 'updated', 'error': 'error' }; + const mock_error = { + response: { + status: 404, + headers: { + 'content-type': 'application/json' + }, + data: JSON.stringify(mock_payload), + statusText: 'Not Found' + } + }; + + mocked_axios.request.mockRejectedValue(mock_error); + + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'POST' + }, new StoryOutputs()); + + expect(result).toStrictEqual({ + status: 404, + content_type: 'application/json', + payload: mock_payload, + message: 'Not Found' + }); + }); }) describe('deserialize_payload', () => { diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 3857cae52..54d65b60e 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -40,27 +40,27 @@ describe('merged API spec', () => { const responses: any = spec.spec().components?.responses test('is added with required fields', () => { - const schema = responses['info@200'].content['application/json'].schema + const schema = responses.info___200.content['application/json'].schema expect(schema.unevaluatedProperties).toEqual({ not: true, errorMessage: 'property is not defined in the spec' }) }) test('is added when no required fields', () => { - const schema = responses['info@500'].content['application/json'].schema + const schema = responses.info___500.content['application/json'].schema expect(schema.unevaluatedProperties).toEqual({ not: true, errorMessage: 'property is not defined in the spec' }) }) test('is not added to empty object schema', () => { - const schema = responses['info@503'].content['application/json'].schema + const schema = responses.info___503.content['application/json'].schema expect(schema.unevaluatedProperties).toBeUndefined() }) test('is not added when true', () => { - const schema = responses['info@201'].content['application/json'].schema + const schema = responses.info___201.content['application/json'].schema expect(schema.unevaluatedProperties).toEqual(true) }) test('is not added when object', () => { - const schema = responses['info@404'].content['application/json'].schema + const schema = responses.info___404.content['application/json'].schema expect(schema.unevaluatedProperties).toEqual({ type: 'object' }) }) }) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 354fcabe4..399c1359e 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -54,7 +54,7 @@ export function construct_tester_components (spec_path: string): { const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new NoOpResultLogger() - const test_runner = new TestRunner(opensearch_http_client, story_validator, story_evaluator, result_logger) + const test_runner = new TestRunner(opensearch_http_client, story_validator, story_evaluator, result_logger, logger) return { specification, operation_locator, From d17091672f410b4cc72cdff69c6b1d53b07add18 Mon Sep 17 00:00:00 2001 From: Ralph Ursprung <39383228+rursprung@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:23:37 +0100 Subject: [PATCH 72/73] README: fix "validate spec" status badge (#657) the workflow got renamed in commit 2a39edd (#646), thus the link is no longer valid. this was not caught by the "check links" test as the link was still valid until the PR got merged. this currently blocks all other PRs. Signed-off-by: Ralph Ursprung --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca512a1e3..a95f722b8 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Test Tools (Unit)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-unit.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-unit.yml) [![Test Tools (Integration)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-integ.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-tools-integ.yml) [![Test Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-spec.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/test-spec.yml) -[![Validate Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec.yml) +[![Validate Spec](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec-lint.yml/badge.svg)](https://github.com/opensearch-project/opensearch-api-specification/actions/workflows/validate-spec-lint.yml) - [OpenSearch API Specification](#opensearch-api-specification) - [Welcome!](#welcome) From e41257a2edc8497477fa486d644d06e2e9b1343c Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 7 Nov 2024 15:36:38 -0500 Subject: [PATCH 73/73] Added `_search` with `sort: direction`. (#658) Signed-off-by: dblock --- CHANGELOG.md | 1 + spec/schemas/_common.yaml | 6 ++ tests/default/_core/search/search_after.yaml | 39 +++++++++ tests/default/_core/search/sort_by_field.yaml | 79 +++++++++++-------- 4 files changed, 90 insertions(+), 35 deletions(-) create mode 100644 tests/default/_core/search/search_after.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 811c2cb21..c34b38a77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649)) - Added Python and Ruby spec validators ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) - Added verbose output of the story being evaluated ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646)) +- Added `_search` with `sort: direction` ([#658](https://github.com/opensearch-project/opensearch-api-specification/pull/658)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 6450e9c6d..883ac273a 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -558,10 +558,16 @@ components: oneOf: - title: field $ref: '#/components/schemas/Field' + - title: field_with_direction + $ref: '#/components/schemas/FieldWithDirection' - title: field_with_order $ref: '#/components/schemas/FieldWithOrder' - title: options $ref: '#/components/schemas/SortOptions' + FieldWithDirection: + type: object + additionalProperties: + $ref: '#/components/schemas/SortOrder' FieldWithOrder: type: object additionalProperties: diff --git a/tests/default/_core/search/search_after.yaml b/tests/default/_core/search/search_after.yaml new file mode 100644 index 000000000..0da17c906 --- /dev/null +++ b/tests/default/_core/search/search_after.yaml @@ -0,0 +1,39 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with search_after. +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {title: Drive, year: 1960} +chapters: + - synopsis: Search with search_after. + path: /{index}/_search + parameters: + index: movies + request: + payload: + sort: + - year: asc + search_after: + - 1960 + method: POST + response: + status: 200 + payload: + hits: + hits: + - _source: + title: The Cruise + year: 1998 diff --git a/tests/default/_core/search/sort_by_field.yaml b/tests/default/_core/search/sort_by_field.yaml index 4256929ee..ff9329286 100644 --- a/tests/default/_core/search/sort_by_field.yaml +++ b/tests/default/_core/search/sort_by_field.yaml @@ -3,12 +3,12 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test different ways to sort by field. epilogues: - - path: /sorted_movies + - path: /movies method: DELETE status: [200] prologues: - - path: /sorted_movies + - path: /movies method: PUT request: payload: @@ -18,40 +18,25 @@ prologues: type: keyword year: type: integer - - - path: /sorted_movies/_doc - method: POST - parameters: - refresh: true - request: - payload: - title: The Lion King - year: 1994 - status: [201] - - path: /sorted_movies/_doc - method: POST - parameters: - refresh: true - request: - payload: - title: The Lion King - year: 2019 - status: [201] - - path: /sorted_movies/_doc + - path: /_bulk method: POST parameters: refresh: true request: + content_type: application/x-ndjson payload: - title: Frozen - year: 2013 - status: [201] + - {create: {_index: movies}} + - {title: The Lion King, year: 1994} + - {create: {_index: movies}} + - {title: The Lion King, year: 2019} + - {create: {_index: movies}} + - {title: Frozen, year: 2013} chapters: - - synopsis: QueryString Sort by year. + - synopsis: Sort by year. path: /{index}/_search parameters: - index: sorted_movies + index: movies sort: year method: GET response: @@ -69,10 +54,10 @@ chapters: title: The Lion King year: 2019 - - synopsis: QueryString Sort by title:desc. + - synopsis: Sort by title:desc. path: /{index}/_search parameters: - index: sorted_movies + index: movies sort: title:desc method: GET response: @@ -90,10 +75,10 @@ chapters: title: Frozen year: 2013 - - synopsis: QueryString Sort by title:desc,year:desc. + - synopsis: Sort by title:desc,year:desc. path: /{index}/_search parameters: - index: sorted_movies + index: movies sort: title:desc,year:desc method: GET response: @@ -111,10 +96,10 @@ chapters: title: Frozen year: 2013 - - synopsis: Body Sort by [title, year]. + - synopsis: Sort by [title, year]. path: /{index}/_search parameters: - index: sorted_movies + index: movies method: GET request: payload: @@ -136,10 +121,10 @@ chapters: title: The Lion King year: 2019 - - synopsis: Body Sort by year:asc,title:desc. + - synopsis: Sort by year:asc,title:desc. path: /{index}/_search parameters: - index: sorted_movies + index: movies method: GET request: payload: @@ -161,3 +146,27 @@ chapters: title: The Lion King year: 1994 + - synopsis: Sort with direction by year:asc,title:desc. + path: /{index}/_search + parameters: + index: movies + method: GET + request: + payload: + sort: + - title: asc + - year: desc + response: + status: 200 + payload: + hits: + hits: + - _source: + title: Frozen + year: 2013 + - _source: + title: The Lion King + year: 2019 + - _source: + title: The Lion King + year: 1994