diff --git a/.cspell b/.cspell index 080c3937b..8b5575fa8 100644 --- a/.cspell +++ b/.cspell @@ -9,6 +9,7 @@ authc authinfo authtoken autocut +autofollow backendroles backpressure beider @@ -78,6 +79,7 @@ lucene Lucene lycheeverse marvinpinto +memlock metaphone mget millis @@ -173,6 +175,7 @@ tokenfilters translog Translog tubone +ulimits unigrams Unmanaged unmatch diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 3fa404483..627dd5f52 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -73,7 +73,7 @@ jobs: - name: Run Tests run: | npm run test:spec -- \ - --opensearch-insecure \ + --opensearch-insecure --verbose\ --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ --tests=tests/${{ matrix.entry.tests || 'default' }} diff --git a/spec/namespaces/replication.yaml b/spec/namespaces/replication.yaml new file mode 100644 index 000000000..5bb133021 --- /dev/null +++ b/spec/namespaces/replication.yaml @@ -0,0 +1,322 @@ +openapi: 3.1.0 +info: + title: OpenSearch Replication API + description: OpenSearch Replication API + version: 1.0.0 +paths: + /_plugins/_replication/{index}/_start: + put: + operationId: replication.start.0 + x-operation-group: replication.start + x-version-added: '1.1' + description: Initiate replication of an index from the leader cluster to the follower cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#start-replication + parameters: + - $ref: '#/components/parameters/replication.start::path.index' + requestBody: + $ref: '#/components/requestBodies/replication.start' + responses: + '200': + $ref: '#/components/responses/replication.start@200' + /_plugins/_replication/{index}/_stop: + post: + operationId: replication.stop.0 + x-operation-group: replication.stop + x-version-added: '1.1' + description: Terminates replication and converts the follower index to a standard index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#stop-replication + parameters: + - $ref: '#/components/parameters/replication.stop::path.index' + requestBody: + $ref: '#/components/requestBodies/replication.stop' + responses: + '200': + $ref: '#/components/responses/replication.stop@200' + /_plugins/_replication/{index}/_pause: + post: + operationId: replication.pause.0 + x-operation-group: replication.pause + x-version-added: '1.1' + description: Pauses replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#pause-replication + parameters: + - $ref: '#/components/parameters/replication.pause::path.index' + requestBody: + $ref: '#/components/requestBodies/replication.pause' + responses: + '200': + $ref: '#/components/responses/replication.pause@200' + /_plugins/_replication/{index}/_resume: + post: + operationId: replication.resume.0 + x-operation-group: replication.resume + x-version-added: '1.1' + description: Resumes replication of the leader index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#resume-replication + parameters: + - $ref: '#/components/parameters/replication.resume::path.index' + requestBody: + $ref: '#/components/requestBodies/replication.resume' + responses: + '200': + $ref: '#/components/responses/replication.resume@200' + /_plugins/_replication/{index}/_status: + get: + operationId: replication.status.0 + x-operation-group: replication.status + x-version-added: '1.1' + description: Gets the status of index replication. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-replication-status + parameters: + - $ref: '#/components/parameters/replication.status::path.index' + responses: + '200': + $ref: '#/components/responses/replication.status@200' + /_plugins/_replication/leader_stats: + get: + operationId: replication.leader_stats.0 + x-operation-group: replication.leader_stats + x-version-added: '1.1' + description: Gets information about replicated leader indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-leader-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.leader_stats@200' + /_plugins/_replication/follower_stats: + get: + operationId: replication.follower_stats.0 + x-operation-group: replication.follower_stats + x-version-added: '1.1' + description: Gets information about follower (syncing) indexes on a specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-follower-cluster-stats + responses: + '200': + $ref: '#/components/responses/replication.follower_stats@200' + /_plugins/_replication/autofollow_stats: + get: + operationId: replication.autofollow_stats.0 + x-operation-group: replication.autofollow_stats + x-version-added: '1.1' + description: Gets information about auto-follow activity and any replication rules configured on the specified cluster. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#get-auto-follow-stats + responses: + '200': + $ref: '#/components/responses/replication.autofollow_stats@200' + /_plugins/_replication/{index}/_update: + put: + operationId: replication.update_settings.0 + x-operation-group: replication.update_settings + x-version-added: '1.1' + description: Updates settings on the follower index. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#update-settings + parameters: + - $ref: '#/components/parameters/replication.update_settings::path.index' + requestBody: + $ref: '#/components/requestBodies/replication.update_settings' + responses: + '200': + $ref: '#/components/responses/replication.update_settings@200' + /_plugins/_replication/_autofollow: + post: + operationId: replication.create_replication_rule.0 + x-operation-group: replication.create_replication_rule + x-version-added: '1.1' + description: Automatically starts replication on indexes matching a specified pattern. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#create-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.create_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.create_replication_rule@200' + delete: + operationId: replication.delete_replication_rule.0 + x-operation-group: replication.delete_replication_rule + x-version-added: '1.1' + description: Deletes the specified replication rule. + externalDocs: + url: https://opensearch.org/docs/latest/tuning-your-cluster/replication-plugin/api/#delete-replication-rule + requestBody: + $ref: '#/components/requestBodies/replication.delete_replication_rule' + responses: + '200': + $ref: '#/components/responses/replication.delete_replication_rule@200' +components: + parameters: + replication.start::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + replication.stop::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + replication.pause::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + replication.resume::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + replication.status::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + replication.update_settings::path.index: + in: path + name: index + description: Name of the data stream, index, or index alias to perform bulk actions on. + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/IndexName' + style: simple + requestBodies: + replication.start: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Replication' + required: true + replication.stop: + content: + application/json: + schema: + type: object + required: true + replication.pause: + content: + application/json: + schema: + type: object + required: true + replication.resume: + content: + application/json: + schema: + type: object + required: true + replication.update_settings: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/UpdateSettings' + required: true + replication.create_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/CreateReplicationRule' + required: true + replication.delete_replication_rule: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/DeleteReplicationRule' + required: true + responses: + replication.start@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.stop@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.pause@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.resume@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.status@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/Status' + replication.leader_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/LeaderStatus' + replication.follower_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/FollowerStatus' + replication.autofollow_stats@200: + content: + application/json: + schema: + $ref: '../schemas/replication._common.yaml#/components/schemas/AutoFollowStatus' + replication.update_settings@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.create_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean + replication.delete_replication_rule@200: + content: + application/json: + schema: + type: object + properties: + acknowledged: + type: boolean \ No newline at end of file diff --git a/spec/schemas/replication._common.yaml b/spec/schemas/replication._common.yaml new file mode 100644 index 000000000..bdd11c3f3 --- /dev/null +++ b/spec/schemas/replication._common.yaml @@ -0,0 +1,222 @@ +openapi: 3.1.0 +info: + title: Schemas of replication._common category. + description: Schemas of replication._common category. + version: 1.0.0 +paths: {} +components: + schemas: + Replication: + type: object + properties: + leader_alias: + type: string + leader_index: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + UseRoles: + type: object + properties: + leader_cluster_role: + type: string + follower_cluster_role: + type: string + Status: + type: object + properties: + status: + type: string + reason: + type: string + leader_alias: + type: string + leader_index: + type: string + follower_index: + type: string + syncing_details: + $ref: '#/components/schemas/SyncingDetails' + SyncingDetails: + type: object + properties: + leader_checkpoint: + type: integer + follower_checkpoint: + type: integer + seq_no: + type: integer + LeaderStatus: + type: object + properties: + num_replicated_indices: + type: number + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexStatus' + IndexStatus: + type: object + properties: + operations_read: + type: number + translog_size_bytes: + type: number + operations_read_lucene: + type: number + operations_read_translog: + type: number + total_read_time_lucene_millis: + type: number + total_read_time_translog_millis: + type: number + bytes_read: + type: number + FollowerStatus: + type: object + properties: + num_syncing_indices: + type: number + num_bootstrapping_indices: + type: number + num_paused_indices: + type: number + num_failed_indices: + type: number + num_shard_tasks: + type: number + num_index_tasks: + type: number + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + index_stats: + type: object + additionalProperties: + $ref: '#/components/schemas/IndexFollowerStatus' + IndexFollowerStatus: + type: object + properties: + operations_written: + type: number + operations_read: + type: number + failed_read_requests: + type: number + throttled_read_requests: + type: number + failed_write_requests: + type: number + throttled_write_requests: + type: number + follower_checkpoint: + type: number + leader_checkpoint: + type: number + total_write_time_millis: + type: number + AutoFollowStatus: + type: object + properties: + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + autofollow_stats: + type: array + items: + $ref: '#/components/schemas/AutoFollowStats' + AutoFollowStats: + type: object + properties: + name: + type: string + pattern: + type: string + num_success_start_replication: + type: number + num_failed_start_replication: + type: number + num_failed_leader_calls: + type: number + failed_indices: + type: array + items: + type: string + SettingsBody: + type: object + properties: + index: + $ref: '#/components/schemas/IndexSchema' + UpdateSettings: + type: object + properties: + settings: + anyOf: + - type: object + properties: + index.number_of_shards: + type: integer + index.number_of_replicas: + type: integer + - $ref: '#/components/schemas/SettingsBody' + IndexSchema: + type: object + properties: + number_of_shards: + type: integer + number_of_replicas: + type: integer + CreateReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string + pattern: + type: string + use_roles: + $ref: '#/components/schemas/UseRoles' + DeleteReplicationRule: + type: object + properties: + leader_alias: + type: string + name: + type: string \ No newline at end of file diff --git a/tests/replication/replica.yaml b/tests/replication/replica.yaml new file mode 100644 index 000000000..c32e141f4 --- /dev/null +++ b/tests/replication/replica.yaml @@ -0,0 +1,128 @@ +$schema: ../../json_schemas/test_story.schema.yaml + +description: Test to explicitly clear the cursor context + +prologues: + - path: /{index} + method: PUT + parameters: + index: books + request_body: + payload: {} + - path: /_bulk + method: POST + request_body: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: books, _id: book_1392215}} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} + - path: /{index}/_refresh + method: POST + parameters: + index: books +epilogues: + - path: /_plugins/_replication/{index}/_stop + method: POST + parameters: + index: books-names + request_body: + payload: {} + - path: /books-names + method: DELETE + status: [200, 404] + - path: /books + method: DELETE + status: [200, 404] +chapters: + - synopsis: Establish connection. + path: /_cluster/settings + method: PUT + request_body: + payload: + persistent: + cluster: + remote: + my-connection-alias: + seeds: ['172.18.0.3:9300'] + response: + status: 200 + - synopsis: Start replication. + path: /_plugins/_replication/{index}/_start + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request_body: + payload: + leader_alias: my-connection-alias + leader_index: books + use_roles: + leader_cluster_role: leader_role + follower_cluster_role: follower_role + response: + status: 200 + - synopsis: Get replication stats. + path: /_plugins/_replication/{index}/_status + version: '>= 1.1' + method: GET + parameters: + index: books-names + response: + status: 200 + - synopsis: Get leader stats. + path: /_plugins/_replication/leader_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get follower stats. + path: /_plugins/_replication/follower_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Get auto-follow stats. + path: /_plugins/_replication/autofollow_stats + version: '>= 1.1' + method: GET + response: + status: 200 + - synopsis: Update settings. + path: /_plugins/_replication/{index}/_update + version: '>= 1.1' + method: PUT + parameters: + index: books-names + request_body: + payload: + settings: + index.number_of_shards: 4 + index.number_of_replicas: 2 + response: + status: 200 + - synopsis: Create replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: POST + request_body: + payload: + leader_alias: my-connection-alias + name: books + pattern: 'books-*' + use_roles: + leader_cluster_role: leader_role + follower_cluster_role: follower_role + response: + status: 200 + - synopsis: Delete replication rule. + path: /_plugins/_replication/_autofollow + version: '>= 1.1' + method: DELETE + request_body: + payload: + leader_alias: my-connection-alias + name: books + response: + status: 200 \ No newline at end of file diff --git a/tests/snapshot/docker-compose.yml b/tests/snapshot/docker-compose.yml index 86cb0b9b3..f908c89e5 100644 --- a/tests/snapshot/docker-compose.yml +++ b/tests/snapshot/docker-compose.yml @@ -1,8 +1,23 @@ version: '3' services: - opensearch-cluster: + opensearch-node1: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + container_name: opensearch-node1 + environment: + - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - bootstrap.memory_lock=true + - cluster.initial_master_nodes=opensearch-node1,opensearch-node2 + - cluster.name=opensearch-cluster + - discovery.seed_hosts=opensearch-node1,opensearch-node2 + - node.name=opensearch-node1 + - path.repo=/tmp/opensearch/repo + - plugins.index_state_management.job_interval=1 + ulimits: + memlock: + soft: -1 + hard: -1 ports: - 9200:9200 - 9600:9600 @@ -11,3 +26,4 @@ services: - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} - discovery.type=single-node - path.repo=/tmp/opensearch/repo +