From a51361e842edd98978bffc0fcc821651f3e84e98 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Wed, 11 Sep 2024 05:53:09 +1200 Subject: [PATCH] Fix tasks namespace (#520) --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 126 ++---------- spec/namespaces/tasks.yaml | 2 +- spec/schemas/_common.yaml | 192 ++++++++++++++---- spec/schemas/tasks._common.yaml | 81 +++++--- tests/default/tasks/list.yaml | 25 +++ tools/src/_utils/JsonSchemaValidator.ts | 8 +- .../tests/_utils/JsonSchemaValidator.test.ts | 13 ++ 8 files changed, 265 insertions(+), 183 deletions(-) create mode 100644 tests/default/tasks/list.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index a0707cc98..f56751ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -151,6 +151,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551)) - Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561)) - Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) +- Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520)) ### Security diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index e4eae5c8f..c9d5927be 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2719,42 +2719,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - deleted: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - requests_per_second: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - slice_id: - type: number - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - throttled: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - throttled_until: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - version_conflicts: - type: number + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' delete_by_query_rethrottle@200: content: application/json: @@ -2982,42 +2952,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - created: - type: number - deleted: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - requests_per_second: - type: number - slice_id: - type: number - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/EpochTimeUnitMillis' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/EpochTimeUnitMillis' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - updated: - type: number - version_conflicts: - type: number + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' reindex_rethrottle@200: content: application/json: @@ -3178,42 +3118,12 @@ components: content: application/json: schema: - type: object - properties: - batches: - type: number - failures: - type: array - items: - $ref: '../schemas/_common.yaml#/components/schemas/BulkIndexByScrollFailure' - noops: - type: number - deleted: - type: number - requests_per_second: - type: number - retries: - $ref: '../schemas/_common.yaml#/components/schemas/Retries' - task: - $ref: '../schemas/_common.yaml#/components/schemas/TaskId' - timed_out: - type: boolean - took: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - total: - type: number - updated: - type: number - version_conflicts: - type: number - throttled: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' - throttled_until: - $ref: '../schemas/_common.yaml#/components/schemas/Duration' - throttled_until_millis: - $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' + oneOf: + - $ref: '../schemas/_common.yaml#/components/schemas/BulkByScrollResponseBase' + - type: object + properties: + task: + $ref: '../schemas/_common.yaml#/components/schemas/TaskId' update_by_query_rethrottle@200: content: application/json: diff --git a/spec/namespaces/tasks.yaml b/spec/namespaces/tasks.yaml index 013155f8f..a0409b8a6 100644 --- a/spec/namespaces/tasks.yaml +++ b/spec/namespaces/tasks.yaml @@ -90,7 +90,7 @@ components: task: $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskInfo' response: - type: object + $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskResponse' error: $ref: '../schemas/_common.yaml#/components/schemas/ErrorCause' required: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index a33d64dd8..39eca3980 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1420,25 +1420,6 @@ components: type: string enum: - auto - BulkIndexByScrollFailure: - type: object - properties: - cause: - $ref: '#/components/schemas/ErrorCause' - id: - $ref: '#/components/schemas/Id' - index: - $ref: '#/components/schemas/IndexName' - status: - type: number - type: - type: string - required: - - cause - - id - - index - - status - - type Retries: type: object properties: @@ -1948,23 +1929,32 @@ components: items: $ref: '#/components/schemas/NodeRole' NodeRole: - type: string - enum: - - client - - cluster_manager - - coordinating_only - - data - - data_cold - - data_content - - data_frozen - - data_hot - - data_warm - - ingest - - master - - ml - - remote_cluster_client - - transform - - voting_only + oneOf: + - type: string + enum: + - client + - coordinating_only + - data + - data_cold + - data_content + - data_frozen + - data_hot + - data_warm + - ingest + - ml + - remote_cluster_client + - transform + - voting_only + - type: string + enum: + - master + deprecated: true + x-version-deprecated: '2.0' + x-deprecation-message: To promote inclusive language, use 'cluster_manager' instead. + - type: string + enum: + - cluster_manager + x-version-added: '2.0' HttpHeaders: type: object additionalProperties: @@ -1995,11 +1985,7 @@ components: transport_address: $ref: '#/components/schemas/TransportAddress' required: - - attributes - - host - - ip - name - - transport_address RankContainer: type: object properties: @@ -2116,3 +2102,129 @@ components: - expand - fetch - query + BulkByScrollTaskStatus: + type: object + properties: + slice_id: + type: integer + format: int32 + total: + description: The number of documents that were successfully processed. + type: integer + format: int64 + updated: + description: The number of documents that were successfully updated, for example, a document with same ID already existed prior to reindex updating it. + type: integer + format: int64 + created: + description: The number of documents that were successfully created. + type: integer + format: int64 + deleted: + description: The number of documents that were successfully deleted. + type: integer + format: int64 + batches: + description: The number of scroll responses pulled back by the reindex. + type: integer + format: int32 + version_conflicts: + description: The number of version conflicts that reindex hits. + type: integer + format: int64 + noops: + description: The number of documents that were ignored. + type: integer + format: int64 + retries: + $ref: '#/components/schemas/Retries' + throttled_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' + throttled: + $ref: '#/components/schemas/Duration' + requests_per_second: + description: The number of requests per second effectively executed during the reindex. + type: number + format: float + canceled: + type: string + throttled_until_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' + throttled_until: + $ref: '#/components/schemas/Duration' + slices: + type: array + items: + $ref: '#/components/schemas/BulkByScrollTaskStatusOrException' + required: + - batches + - deleted + - noops + - requests_per_second + - retries + - throttled_millis + - throttled_until_millis + - total + - version_conflicts + BulkByScrollTaskStatusOrException: + oneOf: + - title: status + $ref: '#/components/schemas/BulkByScrollTaskStatus' + - title: exception + $ref: '#/components/schemas/ErrorCause' + BulkByScrollResponseBase: + allOf: + - $ref: '#/components/schemas/BulkByScrollTaskStatus' + - type: object + properties: + took: + type: integer + format: int64 + timed_out: + type: boolean + failures: + type: array + items: + $ref: '#/components/schemas/BulkByScrollFailure' + required: + - failures + - timed_out + - took + BulkByScrollFailure: + anyOf: + - $ref: '#/components/schemas/BulkItemResponseFailure' + - $ref: '#/components/schemas/ScrollableHitSourceSearchFailure' + BulkItemResponseFailure: + type: object + properties: + cause: + $ref: '#/components/schemas/ErrorCause' + id: + $ref: '#/components/schemas/Id' + index: + $ref: '#/components/schemas/IndexName' + status: + type: integer + format: int32 + required: + - cause + - index + - status + ScrollableHitSourceSearchFailure: + type: object + properties: + index: + $ref: '#/components/schemas/IndexName' + shard: + type: integer + format: int32 + node: + type: string + status: + type: integer + format: int32 + reason: + $ref: '#/components/schemas/ErrorCause' + required: + - reason + - status diff --git a/spec/schemas/tasks._common.yaml b/spec/schemas/tasks._common.yaml index 8e5399cb4..c5afaccda 100644 --- a/spec/schemas/tasks._common.yaml +++ b/spec/schemas/tasks._common.yaml @@ -21,34 +21,20 @@ components: description: Task information grouped by node, if `group_by` was set to `node` (the default). type: object additionalProperties: - $ref: '#/components/schemas/NodeTasks' + $ref: '#/components/schemas/TaskExecutingNode' tasks: $ref: '#/components/schemas/TaskInfos' - NodeTasks: - type: object - properties: - name: - $ref: '_common.yaml#/components/schemas/NodeId' - transport_address: - $ref: '_common.yaml#/components/schemas/TransportAddress' - host: - $ref: '_common.yaml#/components/schemas/Host' - ip: - $ref: '_common.yaml#/components/schemas/Ip' - roles: - type: array - items: - type: string - attributes: - type: object - additionalProperties: - type: string - tasks: - type: object - additionalProperties: - $ref: '#/components/schemas/TaskInfo' - required: - - tasks + TaskExecutingNode: + allOf: + - $ref: '_common.yaml#/components/schemas/BaseNode' + - type: object + properties: + tasks: + type: object + additionalProperties: + $ref: '#/components/schemas/TaskInfo' + required: + - tasks TaskInfo: type: object properties: @@ -76,8 +62,7 @@ components: start_time_in_millis: $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' status: - description: Task status information can vary wildly from task to task. - type: object + $ref: '#/components/schemas/Status' type: type: string parent_task_id: @@ -93,13 +78,15 @@ components: - type TaskInfos: oneOf: - - type: array + - title: grouped_by_none + type: array items: $ref: '#/components/schemas/TaskInfo' - - type: object + - title: grouped_by_parents + type: object additionalProperties: - $ref: '#/components/schemas/ParentTaskInfo' - ParentTaskInfo: + $ref: '#/components/schemas/TaskGroup' + TaskGroup: allOf: - $ref: '#/components/schemas/TaskInfo' - type: object @@ -107,10 +94,38 @@ components: children: type: array items: - $ref: '#/components/schemas/TaskInfo' + $ref: '#/components/schemas/TaskGroup' GroupBy: type: string enum: - nodes - none - parents + Status: + description: Task status information can vary wildly from task to task. + anyOf: + - $ref: '#/components/schemas/ReplicationTaskStatus' + - $ref: '_common.yaml#/components/schemas/BulkByScrollTaskStatus' + - $ref: '#/components/schemas/PersistentTaskStatus' + - $ref: '#/components/schemas/RawTaskStatus' + ReplicationTaskStatus: + type: object + properties: + phase: + type: string + required: + - phase + PersistentTaskStatus: + type: object + properties: + state: + type: string + required: + - state + RawTaskStatus: + type: object + additionalProperties: + title: metadata + TaskResponse: + anyOf: + - $ref: '_common.yaml#/components/schemas/BulkByScrollResponseBase' diff --git a/tests/default/tasks/list.yaml b/tests/default/tasks/list.yaml new file mode 100644 index 000000000..a687e170c --- /dev/null +++ b/tests/default/tasks/list.yaml @@ -0,0 +1,25 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test tasks list endpoint. +chapters: + - synopsis: List tasks grouped by node. + path: /_tasks + method: GET + parameters: + group_by: nodes + response: + status: 200 + - synopsis: List tasks grouped by parent. + path: /_tasks + method: GET + parameters: + group_by: parents + response: + status: 200 + - synopsis: List tasks grouped by none. + path: /_tasks + method: GET + parameters: + group_by: none + response: + status: 200 \ No newline at end of file diff --git a/tools/src/_utils/JsonSchemaValidator.ts b/tools/src/_utils/JsonSchemaValidator.ts index ff33dbc1f..850468363 100644 --- a/tools/src/_utils/JsonSchemaValidator.ts +++ b/tools/src/_utils/JsonSchemaValidator.ts @@ -38,7 +38,13 @@ export default class JsonSchemaValidator { addFormats(this.ajv); if (options.ajv_errors_opts != null) ajv_errors(this.ajv, options.ajv_errors_opts) for (const keyword of options.additional_keywords ?? []) this.ajv.addKeyword(keyword) - Object.entries(options.reference_schemas ?? {}).forEach(([key, schema]) => this.ajv.addSchema(schema, key)) + Object.entries(options.reference_schemas ?? {}).forEach(([key, schema]) => { + try { + this.ajv.addSchema(schema, key); + } catch (e) { + throw new Error(`Failed to add schema ${key} (${JSON.stringify(schema)}):\n\t${e instanceof Error ? e.message : e as string}`) + } + }) this.errors_parser = new AjvErrorsParser(this.ajv, options.errors_text_opts) if (default_schema) this._validate = this.ajv.compile(default_schema) } diff --git a/tools/tests/_utils/JsonSchemaValidator.test.ts b/tools/tests/_utils/JsonSchemaValidator.test.ts index 474dc1206..83b7c29a8 100644 --- a/tools/tests/_utils/JsonSchemaValidator.test.ts +++ b/tools/tests/_utils/JsonSchemaValidator.test.ts @@ -83,4 +83,17 @@ describe('JsonSchemaValidator', () => { expect(validator.validate_schema(invalid_schema)).toEqual('data/required must be array'); }); + + test('constructing with invalid reference schema throws descriptive error', () => { + const invalid_options = { + reference_schemas: { + '#/components/schemas/Invalid': { + type: 'string', + required: true, // required must be an array + } + } + } + expect(() => new JsonSchemaValidator(schema, invalid_options)) + .toThrow('Failed to add schema #/components/schemas/Invalid ({"type":"string","required":true}):\n\tschema is invalid: data/required must be array'); + }); }); \ No newline at end of file