From 106fd001f3409a9edb54be243c63924a31c5f88e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sat, 3 Aug 2024 22:24:39 +0500 Subject: [PATCH 1/9] replication added Signed-off-by: Tokesh --- spec/namespaces/replication.yaml | 261 ++++++++++++++++++++++++++ spec/schemas/replication._common.yaml | 207 ++++++++++++++++++++ tests/replication/replica.yaml | 39 ++++ 3 files changed, 507 insertions(+) create mode 100644 spec/namespaces/replication.yaml create mode 100644 spec/schemas/replication._common.yaml create mode 100644 tests/replication/replica.yaml diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml new file mode 100644 index 000000000..200dd6356 --- /dev/null +++ b/spec/namespaces/replication.yaml @@ -0,0 +1,261 @@ +openapi: 3.1.0 +info: + title: OpenSearch Replication API + description: OpenSearch Replication API + version: 1.0.0 +paths: + /_plugins/_replication/{index}/_start: + put: + operationId: replication.start.0 + x-operation-group: replication.start + x-version-added: '1.1' + description: Initiate replication of an index from the leader cluster to the follower cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication + requestBody: + $ref: '#/components/requestBodies/replication.start' + responses: + '200': + $ref: '#/components/responses/replication.start@200' + /_plugins/_replication/{index}/_stop: + post: + operationId: replication.stop.0 + x-operation-group: replication.stop + x-version-added: '1.1' + description: Terminates replication and converts the follower index to a standard index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication + requestBody: + $ref: '#/components/requestBodies/replication.stop' + responses: + '200': + $ref: '#/components/responses/replication.stop@200' + /_plugins/_replication/{index}/_pause: + post: + operationId: replication.pause.0 + x-operation-group: replication.pause + x-version-added: '1.1' + description: Pauses replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication + requestBody: + $ref: '#/components/requestBodies/replication.pause' + responses: + '200': + $ref: '#/components/responses/replication.pause@200' + /_plugins/_replication/{index}/_resume: + post: + operationId: replication.resume.0 + x-operation-group: replication.resume + x-version-added: '1.1' + description: Resumes replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication + requestBody: + $ref: '#/components/requestBodies/replication.resume' + responses: + '200': + $ref: '#/components/responses/replication.resume@200' + /_plugins/_replication/{index}/_status: + get: + operationId: replication.status.0 + x-operation-group: replication.status + x-version-added: '1.1' + description: Gets the status of index replication. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status + responses: + '200': + $ref: '#/components/responses/replication.status@200' + /_plugins/_replication/leader_stats: + get: + operationId: replication.leader_stats.0 + x-operation-group: replication.leader_stats + x-version-added: '1.1' + description: Gets information about replicated leader indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-leader-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.leader_stats@200' + /_plugins/_replication/follower_stats: + get: + operationId: replication.follower_stats.0 + x-operation-group: replication.follower_stats + x-version-added: '1.1' + description: Gets information about follower (syncing) indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-follower-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.follower_stats@200' + /_plugins/_replication/autofollow_stats: + get: + operationId: replication.autofollow_stats.0 + x-operation-group: replication.autofollow_stats + x-version-added: '1.1' + description: Gets information about auto-follow activity and any replication rules configured on the specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-auto-follow-stats + responses: + '200': + $ref: '#/components/responses/replication.autofollow_stats@200' + /_plugins/_replication/{follower-index}/_update: + put: + operationId: replication.update_settings.0 + x-operation-group: replication.update_settings + x-version-added: '1.1' + description: Updates settings on the follower index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings + requestBody: + $ref: '#/components/requestBodies/replication.update_settings' + responses: + '200': + $ref: '#/components/responses/replication.update_settings@200' + /_plugins/_replication/_autofollow: + post: + operationId: replication.replication_rule.0 + x-operation-group: replication.replication_rule + x-version-added: '1.1' + description: Automatically starts replication on indexes matching a specified pattern. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#create-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.create_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.create_replication_rule@200' + delete: + operationId: replication.delete_replication_rule.0 + x-operation-group: replication.delete_replication_rule + x-version-added: '1.1' + description: Deletes the specified replication rule. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#delete-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.delete_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.delete_replication_rule@200' +components: + requestBodies: + replication.start: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Replication' + required: true + replication.stop: + content: + application/json: + schema: + type: Object + required: true + replication.pause: + content: + application/json: + schema: + type: Object + required: true + replication.resume: + content: + application/json: + schema: + type: Object + required: true + replication.update_settings: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/UpdateSettings' + required: true + replication.create_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/CreateReplicationRule' + required: true + replication.delete_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/DeleteReplicationRule' + required: true + responses: + replication.start@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.stop@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.pause@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.resume@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.status@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Status' + replication.leader_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/LeaderStatus' + replication.follower_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/FollowerStatus' + replication.autofollow_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/AutoFollowStatus' + replication.update_settings@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.create_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.delete_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean \ No newline at end of file diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml new file mode 100644 index 000000000..94febb860 --- /dev/null +++ b/spec/schemas/replication._common.yaml @@ -0,0 +1,207 @@ +openapi: 3.1.0 +info: + title: Schemas of replication._common category. + description: Schemas of replication._common category. + version: 1.0.0 +paths: {} +components: + schemas: + Replication: + type: object + properties: + leader_alias: + type: string + leader_index: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + UseRoles: + type: object + properties: + leader_cluster_role: + type: string + follower_cluster_role: + type: string + Status: + type: object + properties: + status: + type: string + reason: + type: string + leader_alias: + type: string + leader_index: + type: string + follower_index: + type: string + syncing_details: + $ref: '#/components/schemas/SyncingDetails' + SyncingDetails: + type: object + properties: + leader_checkpoint: + type: integer + follower_checkpoint: + type: integer + seq_no: + type: integer + LeaderStatus: + type: object + properties: + num_replicated_indices: + type: number + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + # index_stats: + # type: Object + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexStatus' + IndexStatus: + type: object + properties: + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + FollowerStatus: + type: object + properties: + num_syncing_indices: + type: number + num_bootstrapping_indices: + type: number + num_paused_indices: + type: number + num_failed_indices: + type: number + num_shard_tasks: + type: number + num_index_tasks: + type: number + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + # index_stats: + # type: Object + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexFollowerStatus' + IndexFollowerStatus: + type: object + properties: + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + AutoFollowStatus: + type: object + properties: + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + autofollow_stats: + type: array + items: + $ref: '#/components/schemas/AutoFollowStats' + AutoFollowStats: + type: object + properties: + name: + type: string # Предполагается, что это строка + pattern: + type: string # Предполагается, что это строка + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + UpdateSettings: + type: object + properties: + settings: + type: string + CreateReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string + pattern: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + DeleteReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string \ No newline at end of file diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml new file mode 100644 index 000000000..54cfde91e --- /dev/null +++ b/tests/replication/replica.yaml @@ -0,0 +1,39 @@ +$schema: ../../json_schemas/test_story.schema.yaml + +description: Test to explicitly clear the cursor context + +# prologues: + # - path: /{index} + # method: PUT + # parameters: + # index: books + # request_body: + # payload: {} + # - path: /_bulk + # method: POST + # request_body: + # content_type: application/x-ndjson + # payload: + # - {create: {_index: books, _id: book_1392214}} + # - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + # - {create: {_index: books, _id: book_1392215}} + # - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + # - path: /{index}/_refresh + # method: POST + # parameters: + # index: books +# epilogues: + # - path: /books + # method: DELETE + # status: [200, 404] +chapters: + - synopsis: Cat with a json response. + path: /_cat/nodes + version: '>= 2.0' + method: GET + parameters: + format: json + response: + status: 200 + payload: + - cluster_manager: '*' \ No newline at end of file From e912e13792b16d42e4b60fecbe0f36936eabfd9c Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sat, 3 Aug 2024 22:30:38 +0500 Subject: [PATCH 2/9] fixing lint and trying test verbose Signed-off-by: Tokesh --- .cspell | 1 + .github/workflows/test-spec.yml | 2 +- spec/schemas/replication._common.yaml | 12 ++++---- tests/replication/replica.yaml | 44 +++++++++++++-------------- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/.cspell b/.cspell index 80670e9b4..fff1b5cef 100644 --- a/.cspell +++ b/.cspell @@ -9,6 +9,7 @@ authc authinfo authtoken autocut +autofollow backendroles backpressure beider diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index c147cbf40..01388d80e 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -56,7 +56,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure \ + --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 94febb860..11dfc175b 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -67,10 +67,10 @@ components: type: number # index_stats: # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexStatus' + # additionalProperties: + # type: object + # properties: + # $ref: '#/components/schemas/IndexStatus' IndexStatus: type: object properties: @@ -169,9 +169,9 @@ components: type: object properties: name: - type: string # Предполагается, что это строка + type: string pattern: - type: string # Предполагается, что это строка + type: string num_success_start_replication: type: number num_failed_start_replication: diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 54cfde91e..6a580f21b 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -3,29 +3,29 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context # prologues: - # - path: /{index} - # method: PUT - # parameters: - # index: books - # request_body: - # payload: {} - # - path: /_bulk - # method: POST - # request_body: - # content_type: application/x-ndjson - # payload: - # - {create: {_index: books, _id: book_1392214}} - # - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - # - {create: {_index: books, _id: book_1392215}} - # - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} - # - path: /{index}/_refresh - # method: POST - # parameters: - # index: books +# - path: /{index} +# method: PUT +# parameters: +# index: books +# request_body: +# payload: {} +# - path: /_bulk +# method: POST +# request_body: +# content_type: application/x-ndjson +# payload: +# - {create: {_index: books, _id: book_1392214}} +# - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} +# - {create: {_index: books, _id: book_1392215}} +# - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} +# - path: /{index}/_refresh +# method: POST +# parameters: +# index: books # epilogues: - # - path: /books - # method: DELETE - # status: [200, 404] +# - path: /books +# method: DELETE +# status: [200, 404] chapters: - synopsis: Cat with a json response. path: /_cat/nodes From bd877a8211b359a6ec143bcaebd70a67882a42ae Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 00:16:47 +0500 Subject: [PATCH 3/9] changing docker compose Signed-off-by: Tokesh --- .github/opensearch-cluster/docker-compose.yml | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index bdd6f2be9..3828f452f 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -1,13 +1,57 @@ version: '3' services: - opensearch-cluster: + opensearch-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node1 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - path.repo=/tmp/opensearch/repo + - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data1:/usr/share/opensearch/data ports: - - '9200:9200' - - '9600:9600' + - "9200:9200" + - "9600:9600" + networks: + - opensearch-net + + opensearch-node2: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node2 environment: + - cluster.name=opensearch-cluster + - node.name=opensearch-node2 + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - bootstrap.memory_lock=true + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' - - discovery.type=single-node - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - data2:/usr/share/opensearch/data + networks: + - opensearch-net + +volumes: + data1: + data2: + +networks: + opensearch-net: + driver: bridge From addcd769cdceeb7908d8b32b68c48d55dfcc43c3 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:43:52 +0500 Subject: [PATCH 4/9] adding tests for replication namespace Signed-off-by: Tokesh --- CHANGELOG.md | 1 + spec/namespaces/replication.yaml | 65 +++++++++++- spec/schemas/replication._common.yaml | 39 ++++--- tests/replication/replica.yaml | 147 +++++++++++++++++++++----- 4 files changed, 208 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e24e482b8..0394299b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452)) - Added `/_plugins/_query/settings` ([#456](https://github.com/opensearch-project/opensearch-api-specification/pull/456)) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) +- Added `replication` namespace specs ([464](https://github.com/opensearch-project/opensearch-api-specification/pull/464)) ### Changed diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index 200dd6356..5347131f9 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -12,6 +12,8 @@ paths: description: Initiate replication of an index from the leader cluster to the follower cluster. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication + parameters: + - $ref: '#/components/parameters/start::path.index' requestBody: $ref: '#/components/requestBodies/replication.start' responses: @@ -25,6 +27,8 @@ paths: description: Terminates replication and converts the follower index to a standard index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication + parameters: + - $ref: '#/components/parameters/stop::path.index' requestBody: $ref: '#/components/requestBodies/replication.stop' responses: @@ -38,6 +42,8 @@ paths: description: Pauses replication of the leader index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication + parameters: + - $ref: '#/components/parameters/pause::path.index' requestBody: $ref: '#/components/requestBodies/replication.pause' responses: @@ -51,6 +57,8 @@ paths: description: Resumes replication of the leader index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication + parameters: + - $ref: '#/components/parameters/resume::path.index' requestBody: $ref: '#/components/requestBodies/replication.resume' responses: @@ -64,6 +72,8 @@ paths: description: Gets the status of index replication. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status + parameters: + - $ref: '#/components/parameters/status::path.index' responses: '200': $ref: '#/components/responses/replication.status@200' @@ -100,7 +110,7 @@ paths: responses: '200': $ref: '#/components/responses/replication.autofollow_stats@200' - /_plugins/_replication/{follower-index}/_update: + /_plugins/_replication/{index}/_update: put: operationId: replication.update_settings.0 x-operation-group: replication.update_settings @@ -108,6 +118,8 @@ paths: description: Updates settings on the follower index. externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings + parameters: + - $ref: '#/components/parameters/update_settings::path.index' requestBody: $ref: '#/components/requestBodies/replication.update_settings' responses: @@ -139,6 +151,55 @@ paths: '200': $ref: '#/components/responses/replication.delete_replication_rule@200' components: + parameters: + start::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + stop::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + pause::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + resume::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + status::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + update_settings::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple requestBodies: replication.start: content: @@ -150,7 +211,7 @@ components: content: application/json: schema: - type: Object + type: object required: true replication.pause: content: diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 11dfc175b..6eea148bc 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -65,12 +65,10 @@ components: type: number bytes_read: type: number - # index_stats: - # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexStatus' + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexStatus' IndexStatus: type: object properties: @@ -121,12 +119,10 @@ components: type: number total_write_time_millis: type: number - # index_stats: - # type: Object - # additionalProperties: - # type: object - # properties: - # $ref: '#/components/schemas/IndexFollowerStatus' + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexFollowerStatus' IndexFollowerStatus: type: object properties: @@ -186,7 +182,24 @@ components: type: object properties: settings: - type: string + anyOf: + - type: object + properties: + index.number_of_shards: + type: integer + index.number_of_replicas: + type: integer + - $ref: '#/components/schemas/SettingsBody' + SettingsBody: + type: object + properties: + index: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer CreateReplicationRule: type: object properties: diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index 6a580f21b..afcd07422 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -2,38 +2,127 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context -# prologues: -# - path: /{index} -# method: PUT -# parameters: -# index: books -# request_body: -# payload: {} -# - path: /_bulk -# method: POST -# request_body: -# content_type: application/x-ndjson -# payload: -# - {create: {_index: books, _id: book_1392214}} -# - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} -# - {create: {_index: books, _id: book_1392215}} -# - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} -# - path: /{index}/_refresh -# method: POST -# parameters: -# index: books -# epilogues: -# - path: /books -# method: DELETE -# status: [200, 404] +prologues: +- path: /{index} + method: PUT + parameters: + index: books + request_body: + payload: {} +- path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} +- path: /{index}/_refresh + method: POST + parameters: + index: books +epilogues: +- path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request_body: + payload: {} +- path: /books-names + method: DELETE + status: [200, 404] +- path: /books + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. - path: /_cat/nodes - version: '>= 2.0' + - synopsis: Establish connection. + path: /_cluster/settings + method: PUT + request_body: + payload: + persistent: + cluster: + remote: + my-connection-alias: + seeds: ["172.18.0.3:9300"] + response: + status: 200 + - synopsis: Start replication. + path: /_plugins/_replication/{index}/_start + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request_body: + payload: + leader_alias: 'my-connection-alias' + leader_index: 'books' + use_roles: + leader_cluster_role: 'leader_role' + follower_cluster_role: 'follower_role' + response: + status: 200 + - synopsis: Get replication stats. + path: /_plugins/_replication/{index}/_status + version: '>= 1.1' + method: GET + parameters: + index: books-names + response: + status: 200 + - synopsis: Get leader stats. + path: /_plugins/_replication/leader_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get follower stats. + path: /_plugins/_replication/follower_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + version: '>= 1.1' method: GET + response: + status: 200 + - synopsis: Update settings. + path: /_plugins/_replication/{index}/_update + version: '>= 1.1' + method: PUT parameters: - format: json + index: books-names + request_body: + payload: + settings: + index.number_of_shards: 4 + index.number_of_replicas: 2 + response: + status: 200 + - synopsis: Create replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: POST + request_body: + payload: + leader_alias: "my-connection-alias" + name: "books" + pattern: "books-*" + use_roles: + leader_cluster_role: 'leader_role' + follower_cluster_role: 'follower_role' response: status: 200 + - synopsis: Delete replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: DELETE + request_body: payload: - - cluster_manager: '*' \ No newline at end of file + leader_alias: "my-connection-alias" + name: "books" + response: + status: 200 \ No newline at end of file From 638d60e4978601537df10638e74bf5a72379bc0e Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:50:44 +0500 Subject: [PATCH 5/9] fixing lint, validate specs ci Signed-off-by: Tokesh --- .cspell | 2 + .github/opensearch-cluster/docker-compose.yml | 28 +++--- spec/namespaces/replication.yaml | 24 +++--- tests/replication/replica.yaml | 86 +++++++++---------- 4 files changed, 69 insertions(+), 71 deletions(-) diff --git a/.cspell b/.cspell index fff1b5cef..bc33abec4 100644 --- a/.cspell +++ b/.cspell @@ -79,6 +79,7 @@ lucene Lucene lycheeverse marvinpinto +memlock metaphone mget millis @@ -174,6 +175,7 @@ tokenfilters translog Translog tubone +ulimits unigrams Unmanaged unmatch diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index 3828f452f..cdf77a035 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -5,13 +5,13 @@ services: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} container_name: opensearch-node1 environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - cluster.name=opensearch-cluster - - node.name=opensearch-node1 - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - node.name=opensearch-node1 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 ulimits: @@ -21,8 +21,8 @@ services: volumes: - data1:/usr/share/opensearch/data ports: - - "9200:9200" - - "9600:9600" + - '9200:9200' + - '9600:9600' networks: - opensearch-net @@ -30,13 +30,13 @@ services: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} container_name: opensearch-node2 environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - cluster.name=opensearch-cluster - - node.name=opensearch-node2 - discovery.seed_hosts=opensearch-node1,opensearch-node2 - - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 - - bootstrap.memory_lock=true - - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - node.name=opensearch-node2 - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 ulimits: @@ -48,10 +48,6 @@ services: networks: - opensearch-net -volumes: - data1: - data2: - networks: opensearch-net: driver: bridge diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index 5347131f9..c08d4092e 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -13,7 +13,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication parameters: - - $ref: '#/components/parameters/start::path.index' + - $ref: '#/components/parameters/replication.start::path.index' requestBody: $ref: '#/components/requestBodies/replication.start' responses: @@ -28,7 +28,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication parameters: - - $ref: '#/components/parameters/stop::path.index' + - $ref: '#/components/parameters/replication.stop::path.index' requestBody: $ref: '#/components/requestBodies/replication.stop' responses: @@ -43,7 +43,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication parameters: - - $ref: '#/components/parameters/pause::path.index' + - $ref: '#/components/parameters/replication.pause::path.index' requestBody: $ref: '#/components/requestBodies/replication.pause' responses: @@ -58,7 +58,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication parameters: - - $ref: '#/components/parameters/resume::path.index' + - $ref: '#/components/parameters/replication.resume::path.index' requestBody: $ref: '#/components/requestBodies/replication.resume' responses: @@ -73,7 +73,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status parameters: - - $ref: '#/components/parameters/status::path.index' + - $ref: '#/components/parameters/replication.status::path.index' responses: '200': $ref: '#/components/responses/replication.status@200' @@ -119,7 +119,7 @@ paths: externalDocs: url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings parameters: - - $ref: '#/components/parameters/update_settings::path.index' + - $ref: '#/components/parameters/replication.update_settings::path.index' requestBody: $ref: '#/components/requestBodies/replication.update_settings' responses: @@ -152,7 +152,7 @@ paths: $ref: '#/components/responses/replication.delete_replication_rule@200' components: parameters: - start::path.index: + replication.start::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -160,7 +160,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - stop::path.index: + replication.stop::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -168,7 +168,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - pause::path.index: + replication.pause::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -176,7 +176,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - resume::path.index: + replication.resume::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -184,7 +184,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - status::path.index: + replication.status::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. @@ -192,7 +192,7 @@ components: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndexName' style: simple - update_settings::path.index: + replication.update_settings::path.index: in: path name: index description: Name of the data stream, index, or index alias to perform bulk actions on. diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml index afcd07422..c32e141f4 100644 --- a/tests/replication/replica.yaml +++ b/tests/replication/replica.yaml @@ -3,38 +3,38 @@ $schema: ../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context prologues: -- path: /{index} - method: PUT - parameters: - index: books - request_body: - payload: {} -- path: /_bulk - method: POST - request_body: - content_type: application/x-ndjson - payload: - - {create: {_index: books, _id: book_1392214}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - - {create: {_index: books, _id: book_1392215}} - - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} -- path: /{index}/_refresh - method: POST - parameters: - index: books + - path: /{index} + method: PUT + parameters: + index: books + request_body: + payload: {} + - path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /{index}/_refresh + method: POST + parameters: + index: books epilogues: -- path: /_plugins/_replication/{index}/_stop - method: POST - parameters: - index: books-names - request_body: - payload: {} -- path: /books-names - method: DELETE - status: [200, 404] -- path: /books - method: DELETE - status: [200, 404] + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request_body: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] chapters: - synopsis: Establish connection. path: /_cluster/settings @@ -45,7 +45,7 @@ chapters: cluster: remote: my-connection-alias: - seeds: ["172.18.0.3:9300"] + seeds: ['172.18.0.3:9300'] response: status: 200 - synopsis: Start replication. @@ -56,11 +56,11 @@ chapters: index: books-names request_body: payload: - leader_alias: 'my-connection-alias' - leader_index: 'books' + leader_alias: my-connection-alias + leader_index: books use_roles: - leader_cluster_role: 'leader_role' - follower_cluster_role: 'follower_role' + leader_cluster_role: leader_role + follower_cluster_role: follower_role response: status: 200 - synopsis: Get replication stats. @@ -108,12 +108,12 @@ chapters: method: POST request_body: payload: - leader_alias: "my-connection-alias" - name: "books" - pattern: "books-*" + leader_alias: my-connection-alias + name: books + pattern: 'books-*' use_roles: - leader_cluster_role: 'leader_role' - follower_cluster_role: 'follower_role' + leader_cluster_role: leader_role + follower_cluster_role: follower_role response: status: 200 - synopsis: Delete replication rule. @@ -122,7 +122,7 @@ chapters: method: DELETE request_body: payload: - leader_alias: "my-connection-alias" - name: "books" + leader_alias: my-connection-alias + name: books response: status: 200 \ No newline at end of file From a6e0a87da98b4eef04c195361d6d4e7dea2af5fd Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 01:55:13 +0500 Subject: [PATCH 6/9] fixing docker-compose and validate specs ci Signed-off-by: Tokesh --- .github/opensearch-cluster/docker-compose.yml | 4 ---- spec/namespaces/replication.yaml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index cdf77a035..bfb05f01f 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -18,8 +18,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data1:/usr/share/opensearch/data ports: - '9200:9200' - '9600:9600' @@ -43,8 +41,6 @@ services: memlock: soft: -1 hard: -1 - volumes: - - data2:/usr/share/opensearch/data networks: - opensearch-net diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index c08d4092e..b4f81902e 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -127,8 +127,8 @@ paths: $ref: '#/components/responses/replication.update_settings@200' /_plugins/_replication/_autofollow: post: - operationId: replication.replication_rule.0 - x-operation-group: replication.replication_rule + operationId: replication.create_replication_rule.0 + x-operation-group: replication.create_replication_rule x-version-added: '1.1' description: Automatically starts replication on indexes matching a specified pattern. externalDocs: From b012e7b7eed02e31e34e1bb90530b22a337083d5 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:03:35 +0500 Subject: [PATCH 7/9] fixing validate specs ci Signed-off-by: Tokesh --- spec/namespaces/replication.yaml | 4 ++-- spec/schemas/replication._common.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml index b4f81902e..5bb133021 100644 --- a/spec/namespaces/replication.yaml +++ b/spec/namespaces/replication.yaml @@ -217,13 +217,13 @@ components: content: application/json: schema: - type: Object + type: object required: true replication.resume: content: application/json: schema: - type: Object + type: object required: true replication.update_settings: content: diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index 6eea148bc..d33ff2b0d 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -183,13 +183,13 @@ components: properties: settings: anyOf: + - $ref: '#/components/schemas/SettingsBody' - type: object properties: index.number_of_shards: type: integer index.number_of_replicas: type: integer - - $ref: '#/components/schemas/SettingsBody' SettingsBody: type: object properties: From 236fea70ebb30d5b200d78f560509085428a0d66 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:06:25 +0500 Subject: [PATCH 8/9] trying to fix validate specs Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index d33ff2b0d..be864a967 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -178,28 +178,28 @@ components: type: array items: type: string + SettingsBody: + type: object + properties: + index: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer UpdateSettings: type: object properties: settings: anyOf: - - $ref: '#/components/schemas/SettingsBody' - type: object properties: index.number_of_shards: type: integer index.number_of_replicas: type: integer - SettingsBody: - type: object - properties: - index: - type: object - properties: - number_of_shards: - type: integer - number_of_replicas: - type: integer + - $ref: '#/components/schemas/SettingsBody' CreateReplicationRule: type: object properties: From 864ee68e743e209038ffd1062a1e0d9131e919f8 Mon Sep 17 00:00:00 2001 From: Tokesh Date: Sun, 4 Aug 2024 02:08:36 +0500 Subject: [PATCH 9/9] trying to fix validate specs v2 Signed-off-by: Tokesh --- spec/schemas/replication._common.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml index be864a967..bdd11c3f3 100644 --- a/spec/schemas/replication._common.yaml +++ b/spec/schemas/replication._common.yaml @@ -182,12 +182,7 @@ components: type: object properties: index: - type: object - properties: - number_of_shards: - type: integer - number_of_replicas: - type: integer + $ref: '#/components/schemas/IndexSchema' UpdateSettings: type: object properties: @@ -200,6 +195,13 @@ components: index.number_of_replicas: type: integer - $ref: '#/components/schemas/SettingsBody' + IndexSchema: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer CreateReplicationRule: type: object properties: