Skip to content

Commit

Permalink
Added/fixed /_plugins/_rollup/jobs. (#716)
Browse files Browse the repository at this point in the history
Signed-off-by: dblock <[email protected]>
Signed-off-by: Thomas Farr <[email protected]>
Co-authored-by: Thomas Farr <[email protected]>
  • Loading branch information
dblock and Xtansia authored Dec 10, 2024
1 parent a54a5f5 commit c0237fb
Show file tree
Hide file tree
Showing 7 changed files with 279 additions and 48 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added `sm` namespace API specifications ([#701](https://github.com/opensearch-project/opensearch-api-specification/pull/701))
- Added schema for `/_plugins/_knn/stats`, `/_plugins/_knn/models/{model_id}`, `_train` and `_search` ([#704](https://github.com/opensearch-project/opensearch-api-specification/pull/704))
- Added `retry` support in `prologues` and `epilogues` ([#713](https://github.com/opensearch-project/opensearch-api-specification/pull/713))
- Added response schema for `DELETE /_plugins/_rollup/jobs/{id}`, `POST /_plugins/_rollup/jobs/{id}/_start` and `_stop` ([#716](https://github.com/opensearch-project/opensearch-api-specification/pull/716))

### Removed
- Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
Expand Down Expand Up @@ -60,6 +61,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Fixed query DSL schemas ([#706](https://github.com/opensearch-project/opensearch-api-specification/pull/706))
- Fixed content-type of `GET /_plugins/_observability/_local/stats` ([#711](https://github.com/opensearch-project/opensearch-api-specification/pull/711))
- Fixed `tenant` in `ObservabilityObject` request body to not be required ([#711](https://github.com/opensearch-project/opensearch-api-specification/pull/711))
- Fixed response code in `PUT /_plugins/_rollup/jobs/{id}` ([#716](https://github.com/opensearch-project/opensearch-api-specification/pull/716))

### Changed
- Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))
Expand Down
24 changes: 18 additions & 6 deletions spec/namespaces/rollups.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@ paths:
externalDocs:
url: https://opensearch.org/docs/latest/im-plugin/index-rollups/rollup-api/#create-or-update-an-index-rollup-job
responses:
'200':
$ref: '#/components/responses/rollups.put@200'
'201':
$ref: '#/components/responses/rollups.put@201'
parameters:
- $ref: '#/components/parameters/rollups.put::path.id'
- $ref: '#/components/parameters/rollups.put::query.if_primary_term'
Expand Down Expand Up @@ -97,14 +97,26 @@ components:
application/json:
schema:
$ref: '../schemas/rollups._common.yaml#/components/schemas/RollupEntity'
rollups.put@200:
rollups.put@201:
content:
application/json:
schema:
$ref: '../schemas/rollups._common.yaml#/components/schemas/RollupEntity'
rollups.delete@200: {}
rollups.start@200: {}
rollups.stop@200: {}
rollups.delete@200:
content:
application/json:
schema:
$ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase'
rollups.start@200:
content:
application/json:
schema:
$ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase'
rollups.stop@200:
content:
application/json:
schema:
$ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase'
rollups.explain@200:
content:
application/json:
Expand Down
59 changes: 17 additions & 42 deletions spec/schemas/rollups._common.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,14 @@ components:
type: object
properties:
_id:
type: string
_seqNo:
type: number
_primaryTerm:
type: number
$ref: '_common.yaml#/components/schemas/Id'
_seq_no:
$ref: '_common.yaml#/components/schemas/SequenceNumber'
_primary_term:
type: integer
format: int64
_version:
$ref: '_common.yaml#/components/schemas/VersionNumber'
rollup:
$ref: '#/components/schemas/Rollup'
Rollup:
Expand All @@ -39,7 +42,7 @@ components:
target_index:
type: string
metadata_id:
type: string
type: ['null', string]
page_size:
type: number
delay:
Expand All @@ -59,18 +62,15 @@ components:
description: 'Set up a Mustache message template for error notifications. For example, if an index rollup job fails, the system sends a message to a Slack channel.'
ExplainEntities:
type: object
properties:
item:
$ref: '#/components/schemas/Explain'
additionalProperties:
$ref: '#/components/schemas/Explain'
Explain:
type: object
properties:
rollup_id:
type: string
last_updated_time:
type: number
continuous:
$ref: '#/components/schemas/Continuous'
metadata_id:
type: ['null', string]
rollup_metadata:
type: ['null', object]
DimensionsConfigItem:
type: object
properties:
Expand All @@ -95,6 +95,8 @@ components:
type: string
target_field:
type: string
format:
type: ['null', string]
HistogramDimension:
type: object
properties:
Expand Down Expand Up @@ -166,30 +168,3 @@ components:
- $ref: '#/components/schemas/Cron'
schedule_delay:
type: number
Continuous:
type: object
properties:
next_window_start_time:
type: number
next_window_end_time:
type: number
status:
type: string
failure_reason:
type: string
stats:
$ref: '#/components/schemas/Stats'
Stats:
type: object
properties:
pages_processed:
type: number
documents_processed:
type: number
rollups_indexed:
type: number
index_time_in_ms:
type: number
search_time_in_ms:
type: number

67 changes: 67 additions & 0 deletions tests/default/rollups/jobs.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@

$schema: ../../../json_schemas/test_story.schema.yaml

description: Test index rollups.

prologues:
- path: /_bulk
method: POST
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {create: {_index: books, _id: book_1392214}}
- {author: Harper Lee, title: To Kill a Mockingbird, order_date: '2021-02-27T03:56:10+00:00', pages: 285}
- {create: {_index: books, _id: book_1392215}}
- {author: Elizabeth Rudnick, title: Beauty and the Beast, order_date: '2024-12-09T04:37:10-05:00', pages: 275}
epilogues:
- path: /_plugins/_rollup/jobs/books
method: DELETE
status: [200, 404]
- path: /books
method: DELETE
status: [200, 404]
chapters:
- synopsis: Create an index rollup job.
path: /_plugins/_rollup/jobs/{id}
method: PUT
parameters:
id: books
request:
payload:
rollup:
enabled: true
description: Rollup books.
schedule:
interval:
period: 1
unit: Minutes
source_index: books
target_index: books_by_order_date
page_size: 1
delay: 0
continuous: false
dimensions:
- date_histogram:
source_field: order_date
fixed_interval: 1h
- terms:
source_field: title
metrics:
- source_field: pages
metrics:
- sum: {}
- avg: {}
response:
status: 201
payload:
_id: books
rollup:
rollup_id: books
enabled: true
- synopsis: Delete an index rollup job.
path: /_plugins/_rollup/jobs/{id}
method: DELETE
parameters:
id: books
59 changes: 59 additions & 0 deletions tests/default/rollups/jobs/explain.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@

$schema: ../../../../json_schemas/test_story.schema.yaml

description: Test index rollups.

prologues:
- path: /_bulk
method: POST
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {create: {_index: books, _id: book_1392214}}
- {author: Harper Lee, title: To Kill a Mockingbird, order_date: '2021-02-27T03:56:10+00:00', pages: 285}
- {create: {_index: books, _id: book_1392215}}
- {author: Elizabeth Rudnick, title: Beauty and the Beast, order_date: '2024-12-09T04:37:10-05:00', pages: 275}
- path: /_plugins/_rollup/jobs/books_rollup_job
method: PUT
request:
payload:
rollup:
enabled: true
description: Rollup books.
schedule:
interval:
period: 1
unit: Minutes
source_index: books
target_index: books_by_order_date
page_size: 1
delay: 0
continuous: false
dimensions:
- date_histogram:
source_field: order_date
fixed_interval: 1h
- terms:
source_field: title
metrics:
- source_field: pages
metrics:
- sum: {}
- avg: {}
- path: /_plugins/_rollup/jobs/books_rollup_job/_start
method: POST
epilogues:
- path: /_plugins/_rollup/jobs/books_rollup_job
method: DELETE
status: [200, 404]
- path: /books
method: DELETE
status: [200, 404]
chapters:
- synopsis: Explain an index rollup job.
path: /_plugins/_rollup/jobs/{id}/_explain
method: GET
parameters:
id: books_rollup_job
57 changes: 57 additions & 0 deletions tests/default/rollups/jobs/start.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@

$schema: ../../../../json_schemas/test_story.schema.yaml

description: Test index rollups.

prologues:
- path: /_bulk
method: POST
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {create: {_index: books, _id: book_1392214}}
- {author: Harper Lee, title: To Kill a Mockingbird, order_date: '2021-02-27T03:56:10+00:00', pages: 285}
- {create: {_index: books, _id: book_1392215}}
- {author: Elizabeth Rudnick, title: Beauty and the Beast, order_date: '2024-12-09T04:37:10-05:00', pages: 275}
- path: /_plugins/_rollup/jobs/books
method: PUT
request:
payload:
rollup:
enabled: true
description: Rollup books.
schedule:
interval:
period: 1
unit: Minutes
source_index: books
target_index: books_by_order_date
page_size: 1
delay: 0
continuous: false
dimensions:
- date_histogram:
source_field: order_date
fixed_interval: 1h
- terms:
source_field: title
metrics:
- source_field: pages
metrics:
- sum: {}
- avg: {}
epilogues:
- path: /_plugins/_rollup/jobs/books
method: DELETE
status: [200, 404]
- path: /books
method: DELETE
status: [200, 404]
chapters:
- synopsis: Start an index rollup job.
path: /_plugins/_rollup/jobs/{id}/_start
method: POST
parameters:
id: books
59 changes: 59 additions & 0 deletions tests/default/rollups/jobs/stop.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@

$schema: ../../../../json_schemas/test_story.schema.yaml

description: Test index rollups.

prologues:
- path: /_bulk
method: POST
parameters:
refresh: true
request:
content_type: application/x-ndjson
payload:
- {create: {_index: books, _id: book_1392214}}
- {author: Harper Lee, title: To Kill a Mockingbird, order_date: '2021-02-27T03:56:10+00:00', pages: 285}
- {create: {_index: books, _id: book_1392215}}
- {author: Elizabeth Rudnick, title: Beauty and the Beast, order_date: '2024-12-09T04:37:10-05:00', pages: 275}
- path: /_plugins/_rollup/jobs/books
method: PUT
request:
payload:
rollup:
enabled: true
description: Rollup books.
schedule:
interval:
period: 1
unit: Minutes
source_index: books
target_index: books_by_order_date
page_size: 1
delay: 0
continuous: false
dimensions:
- date_histogram:
source_field: order_date
fixed_interval: 1h
- terms:
source_field: title
metrics:
- source_field: pages
metrics:
- sum: {}
- avg: {}
- path: /_plugins/_rollup/jobs/books/_start
method: POST
epilogues:
- path: /_plugins/_rollup/jobs/books
method: DELETE
status: [200, 404]
- path: /books
method: DELETE
status: [200, 404]
chapters:
- synopsis: Stop an index rollup job.
path: /_plugins/_rollup/jobs/{id}/_stop
method: POST
parameters:
id: books

0 comments on commit c0237fb

Please sign in to comment.