From 13a603c661806744f431f4cf265ddbdab1c40a54 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 4 Aug 2024 18:50:11 -0400 Subject: [PATCH 01/31] Added OpenSearch 3.0. (#459) * Added OpenSearch 3.0. Signed-off-by: dblock * Removed unneeded quotes. Signed-off-by: dblock * Add --verbose to debug SQL failure. Signed-off-by: dblock * Increase plugin memory circuit breaker limit. Signed-off-by: dblock * Make pretty. Signed-off-by: dblock * Remove verbose. Signed-off-by: dblock * Get and set container ID as output. Signed-off-by: dblock --------- Signed-off-by: dblock --- .github/opensearch-cluster/docker-compose.yml | 6 ++--- .github/workflows/test-spec.yml | 24 +++++++++++++++---- CHANGELOG.md | 1 + spec/schemas/nodes._common.yaml | 3 +++ tests/ingest/pipeline.yaml | 2 +- tests/sql/close.yaml | 12 +++++++--- tests/sql/explain.yaml | 6 +++++ tests/sql/query.yaml | 6 +++++ 8 files changed, 49 insertions(+), 11 deletions(-) diff --git a/.github/opensearch-cluster/docker-compose.yml b/.github/opensearch-cluster/docker-compose.yml index 48a008634..e2ea4b2de 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/.github/opensearch-cluster/docker-compose.yml @@ -4,10 +4,10 @@ services: opensearch-cluster: image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} ports: - - '9200:9200' - - '9600:9600' + - 9200:9200 + - 9600:9600 environment: - - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}' + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} - discovery.type=single-node - path.repo=/tmp/opensearch/repo diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 86590741a..80f7577f8 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -23,12 +23,17 @@ jobs: strategy: matrix: entry: - - {version: 1.3.17, admin_password: admin} - - {version: 2.0.0, admin_password: admin} - - {version: 2.15.0} + - version: 1.3.17 + admin_password: admin + - version: 2.0.0 + admin_password: admin + - version: 2.15.0 - version: 2.16.0 hub: opensearchstaging - ref: '@sha256:50fbfe3b95c41e92a113ada3e80513ba4524dfc8a25dc6aaeff2bbe1e1145d5f' + ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e' + - version: 3.0.0 + hub: opensearchstaging + ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}) runs-on: ubuntu-latest @@ -56,6 +61,11 @@ jobs: working-directory: .github/opensearch-cluster run: docker compose up -d + - name: Get Container ID + id: container + run: | + echo "CONTAINER_ID=$(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}")" >> $GITHUB_OUTPUT + - name: Run Tests run: | npm run test:spec -- \ @@ -63,6 +73,12 @@ jobs: --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json + - name: Get Container Logs + if: failure() && steps.container.outputs.CONTAINER_ID + run: | + echo Dumping logs from ${{ steps.container.outputs.CONTAINER_ID }} ... + docker logs ${{ steps.container.outputs.CONTAINER_ID }} + - name: Upload Test Coverage Results uses: actions/upload-artifact@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e24e482b8..dadcbee4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452)) - Added `/_plugins/_query/settings` ([#456](https://github.com/opensearch-project/opensearch-api-specification/pull/456)) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) +- Added tests against OpenSearch 3.0 ([#459](https://github.com/opensearch-project/opensearch-api-specification/pull/459)) ### Changed diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index ee172b93e..59378e88a 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -1092,6 +1092,9 @@ components: $ref: '#/components/schemas/ShardSearchBackpressureTaskResourceTrackerStats' cancellation_stats: $ref: '#/components/schemas/ShardSearchBackpressureTaskCancellationStats' + completion_count: + x-version-added: '3.0' + type: number ShardSearchBackpressureTaskResourceTrackerStats: type: object properties: diff --git a/tests/ingest/pipeline.yaml b/tests/ingest/pipeline.yaml index c10c4f327..ee8e539ea 100644 --- a/tests/ingest/pipeline.yaml +++ b/tests/ingest/pipeline.yaml @@ -6,7 +6,7 @@ epilogues: - path: /_ingest/pipeline/books_pipeline method: DELETE status: [200, 404] -version: '>= 2.11' +version: '>= 2.11, < 3.0' # TODO: re-enable using a 3.0 build with the neural-search plugin chapters: - synopsis: Create ingest pipeline for text embedding. path: /_ingest/pipeline/{id} diff --git a/tests/sql/close.yaml b/tests/sql/close.yaml index 8df8a4c01..df2910ae9 100644 --- a/tests/sql/close.yaml +++ b/tests/sql/close.yaml @@ -22,7 +22,13 @@ prologues: method: POST parameters: index: books - - id: query_sql + - path: _plugins/_query/settings + method: PUT + request_body: + payload: + transient: + plugins.query.memory_limit: 100% + - id: sql_query path: /_plugins/_sql method: POST request_body: @@ -41,6 +47,6 @@ chapters: method: POST request_body: payload: - cursor: ${query_sql.cursor} + cursor: ${sql_query.cursor} response: - status: 200 \ No newline at end of file + status: 200 diff --git a/tests/sql/explain.yaml b/tests/sql/explain.yaml index 16ea254f8..3679dcb29 100644 --- a/tests/sql/explain.yaml +++ b/tests/sql/explain.yaml @@ -7,6 +7,12 @@ prologues: method: PUT request_body: payload: {} + - path: _plugins/_query/settings + method: PUT + request_body: + payload: + transient: + plugins.query.memory_limit: 100% epilogues: - path: /books method: DELETE diff --git a/tests/sql/query.yaml b/tests/sql/query.yaml index 3111d0d64..e66942606 100644 --- a/tests/sql/query.yaml +++ b/tests/sql/query.yaml @@ -9,6 +9,12 @@ prologues: index: books request_body: payload: {} + - path: _plugins/_query/settings + method: PUT + request_body: + payload: + transient: + plugins.query.memory_limit: 100% epilogues: - path: /books method: DELETE From ac8659de90e65433497dc75c66e002d24e1f63e1 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 4 Aug 2024 19:24:45 -0400 Subject: [PATCH 02/31] Validate titles and descriptions in info and schema objects. (#463) Signed-off-by: dblock --- CHANGELOG.md | 1 + TESTING_GUIDE.md | 4 +- json_schemas/test_story.schema.yaml | 2 +- package-lock.json | 25 ++++ package.json | 2 + spec/namespaces/_core.yaml | 47 +------- spec/namespaces/cat.yaml | 26 +---- spec/namespaces/cluster.yaml | 35 ++---- spec/namespaces/dangling_indices.yaml | 5 +- spec/namespaces/indices.yaml | 55 +-------- spec/namespaces/ingest.yaml | 7 +- spec/namespaces/knn.yaml | 20 ++-- spec/namespaces/ml.yaml | 2 +- spec/namespaces/nodes.yaml | 9 +- spec/namespaces/notifications.yaml | 11 +- spec/namespaces/ppl.yaml | 2 +- spec/namespaces/remote_store.yaml | 5 +- spec/namespaces/rollups.yaml | 11 +- spec/namespaces/search_pipeline.yaml | 5 +- spec/namespaces/security.yaml | 109 +----------------- spec/namespaces/snapshot.yaml | 13 +-- spec/namespaces/sql.yaml | 2 +- spec/namespaces/tasks.yaml | 5 +- spec/namespaces/transforms.yaml | 7 +- spec/schemas/_common.query_dsl.yaml | 2 +- spec/schemas/_common.yaml | 25 ++-- spec/schemas/indices._common.yaml | 4 +- tools/src/linter/components/Info.ts | 47 ++++++++ tools/src/linter/components/NamespaceFile.ts | 13 +++ tools/src/linter/components/Operation.ts | 4 +- tools/src/linter/components/Schema.ts | 12 +- tools/tests/linter/NamespaceFile.test.ts | 54 ++++++++- tools/tests/linter/Operation.test.ts | 2 +- tools/tests/linter/Schema.test.ts | 8 ++ .../tests/linter/factories/namespace_file.ts | 3 + .../namespaces/invalid_info_capitals.yaml | 4 + .../namespaces/invalid_info_periods.yaml | 4 + .../valid_info_multiline_description.yaml | 6 + .../namespaces/invalid_files/cat.yaml | 4 + .../namespaces/invalid_files/dup_path_a.yaml | 4 + .../namespaces/invalid_files/dup_path_b.yaml | 4 + .../namespaces/invalid_files/dup_path_c.yaml | 4 + .../invalid_files/invalid_spec.yaml | 4 + .../namespaces/invalid_folder/cat.yaml | 4 + .../namespaces/invalid_folder/dup_path_a.yaml | 5 + .../namespaces/invalid_folder/dup_path_b.yaml | 4 + .../namespaces/invalid_folder/dup_path_c.yaml | 4 + .../namespaces/invalid_folder/dup_path_d.yaml | 4 + .../invalid_folder/dup_path_namespace_a.yaml | 1 + .../invalid_folder/dup_path_namespace_b.yaml | 1 + .../namespaces/ops.yaml | 4 + .../namespaces/indices.yaml | 4 + .../anonymous_schemas/namespaces/shelter.yaml | 1 - .../named_schemas/namespaces/shelter.yaml | 1 - .../fixtures/extractor/expected_1.3.yaml | 5 - .../fixtures/extractor/expected_2.0.yaml | 5 - .../merger/fixtures/merger/expected.yaml | 2 - .../fixtures/spec/namespaces/indices.yaml | 1 - .../fixtures/spec/namespaces/shelter.yaml | 1 - .../specs/complete/namespaces/index.yaml | 7 +- .../tests/tester/fixtures/specs/excerpt.yaml | 4 - 61 files changed, 288 insertions(+), 388 deletions(-) create mode 100644 tools/src/linter/components/Info.ts create mode 100644 tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_capitals.yaml create mode 100644 tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_periods.yaml create mode 100644 tools/tests/linter/fixtures/file_validators/namespaces/valid_info_multiline_description.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index dadcbee4d..8bb8bbfd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `retry` to `tester` to support asynchronous tasks ([453](https://github.com/opensearch-project/opensearch-api-specification/pull/453)) - Added passing OPENSEARCH_JAVA_OPTS into the docker container used for tests ([#454](https://github.com/opensearch-project/opensearch-api-specification/pull/454)) - Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452)) +- Added validation of titles and descriptions in info and schema objects ([#463](https://github.com/opensearch-project/opensearch-api-specification/pull/463)) - Added `/_plugins/_query/settings` ([#456](https://github.com/opensearch-project/opensearch-api-specification/pull/456)) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) - Added tests against OpenSearch 3.0 ([#459](https://github.com/opensearch-project/opensearch-api-specification/pull/459)) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index efe86d41e..948ce228f 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -115,8 +115,8 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) method: POST request_body: payload: - name: "NLP_Group" - description: "Model group for NLP models" + name: NLP_Group + description: Model group for NLP models. response: status: 200 output: # Save the model group id for later use. diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 74d16fd48..d0b09e5ce 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -94,7 +94,7 @@ definitions: The values are paths to the values in the response. The values should be in the form: - `payload.` for the payload - - `headers.` for the headers + - `headers.` for the headers. type: object additionalProperties: type: string diff --git a/package-lock.json b/package-lock.json index 190f24271..f86e80d30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -20,6 +20,7 @@ "@types/lodash": "^4.14.202", "@types/node": "^20.10.3", "@types/qs": "^6.9.15", + "@types/titlecase": "^1.1.2", "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^6.21.0", "ajv": "^8.13.0", @@ -42,6 +43,7 @@ "lodash": "^4.17.21", "qs": "^6.12.1", "smile-js": "^0.7.0", + "titlecase": "^1.1.3", "tmp": "^0.2.3", "ts-jest": "^29.1.2", "ts-node": "^10.9.1", @@ -2298,6 +2300,11 @@ "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==" }, + "node_modules/@types/titlecase": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/titlecase/-/titlecase-1.1.2.tgz", + "integrity": "sha512-CnYNirLVqAU9E5cu+Y6VwpzBPnOPEt/wFSfi4pkIHesWZPtZWul9aSuLeNwD7lZfR3kpiKHMcgsVyb2mdXNTvg==" + }, "node_modules/@types/tmp": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.6.tgz", @@ -7691,6 +7698,14 @@ "node": ">=0.12" } }, + "node_modules/titlecase": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/titlecase/-/titlecase-1.1.3.tgz", + "integrity": "sha512-pQX4oiemzjBEELPqgK4WE+q0yhAqjp/yzusGtlSJsOuiDys0RQxggepYmo0BuegIDppYS3b3cpdegRwkpyN3hw==", + "bin": { + "to-title-case": "bin.js" + } + }, "node_modules/tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", @@ -10045,6 +10060,11 @@ "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==" }, + "@types/titlecase": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@types/titlecase/-/titlecase-1.1.2.tgz", + "integrity": "sha512-CnYNirLVqAU9E5cu+Y6VwpzBPnOPEt/wFSfi4pkIHesWZPtZWul9aSuLeNwD7lZfR3kpiKHMcgsVyb2mdXNTvg==" + }, "@types/tmp": { "version": "0.2.6", "resolved": "https://registry.npmjs.org/@types/tmp/-/tmp-0.2.6.tgz", @@ -13798,6 +13818,11 @@ "next-tick": "^1.1.0" } }, + "titlecase": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/titlecase/-/titlecase-1.1.3.tgz", + "integrity": "sha512-pQX4oiemzjBEELPqgK4WE+q0yhAqjp/yzusGtlSJsOuiDys0RQxggepYmo0BuegIDppYS3b3cpdegRwkpyN3hw==" + }, "tmp": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", diff --git a/package.json b/package.json index b574d42df..04e7116e4 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@types/lodash": "^4.14.202", "@types/node": "^20.10.3", "@types/qs": "^6.9.15", + "@types/titlecase": "^1.1.2", "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^6.21.0", "ajv": "^8.13.0", @@ -52,6 +53,7 @@ "lodash": "^4.17.21", "qs": "^6.12.1", "smile-js": "^0.7.0", + "titlecase": "^1.1.3", "tmp": "^0.2.3", "ts-jest": "^29.1.2", "ts-node": "^10.9.1", diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 398fe6838..9aa5dbeaa 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: - title: OpenSearch _core API - description: OpenSearch _core API + title: OpenSearch Core API + description: OpenSearch core API. version: 1.0.0 paths: /: @@ -2613,7 +2613,6 @@ components: description: The search definition using the Query DSL responses: bulk@200: - description: '' content: application/json: schema: @@ -2638,7 +2637,6 @@ components: - items - took clear_scroll@200: - description: '' content: application/json: schema: @@ -2652,7 +2650,6 @@ components: - num_freed - succeeded count@200: - description: '' content: application/json: schema: @@ -2666,13 +2663,11 @@ components: - _shards - count create@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' create_pit@200: - description: '' content: application/json: schema: @@ -2686,13 +2681,11 @@ components: type: integer format: int64 delete@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' delete_all_pits@200: - description: '' content: application/json: schema: @@ -2703,7 +2696,6 @@ components: items: $ref: '../schemas/_core._common.yaml#/components/schemas/PitsDetailsDeleteAll' delete_by_query@200: - description: '' content: application/json: schema: @@ -2744,13 +2736,11 @@ components: version_conflicts: type: number delete_by_query_rethrottle@200: - description: '' content: application/json: schema: $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskListResponseBase' delete_pit@200: - description: '' content: application/json: schema: @@ -2761,21 +2751,17 @@ components: items: $ref: '../schemas/_core._common.yaml#/components/schemas/DeletedPit' delete_script@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' exists@200: - description: '' content: application/json: {} exists_source@200: - description: '' content: application/json: {} explain@200: - description: '' content: application/json: schema: @@ -2796,7 +2782,6 @@ components: - _index - matched field_caps@200: - description: '' content: application/json: schema: @@ -2814,13 +2799,11 @@ components: - fields - indices get@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.get.yaml#/components/schemas/GetResult' get_all_pits@200: - description: '' content: application/json: schema: @@ -2831,7 +2814,6 @@ components: items: $ref: '../schemas/_core._common.yaml#/components/schemas/PitDetail' get_script@200: - description: '' content: application/json: schema: @@ -2847,7 +2829,6 @@ components: - _id - found get_script_context@200: - description: '' content: application/json: schema: @@ -2860,7 +2841,6 @@ components: required: - contexts get_script_languages@200: - description: '' content: application/json: schema: @@ -2878,25 +2858,21 @@ components: - language_contexts - types_allowed get_source@200: - description: '' content: application/json: schema: type: object index@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' index@201: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' info@200: - description: '' content: application/json: schema: @@ -2919,7 +2895,6 @@ components: - tagline - version mget@200: - description: '' content: application/json: schema: @@ -2932,19 +2907,16 @@ components: required: - docs msearch@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.msearch.yaml#/components/schemas/MultiSearchResult' msearch_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.msearch.yaml#/components/schemas/MultiSearchResult' mtermvectors@200: - description: '' content: application/json: schema: @@ -2957,17 +2929,14 @@ components: required: - docs ping@200: - description: '' content: application/json: {} put_script@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' rank_eval@200: - description: '' content: application/json: schema: @@ -2990,7 +2959,6 @@ components: - failures - metric_score reindex@200: - description: '' content: application/json: schema: @@ -3031,7 +2999,6 @@ components: version_conflicts: type: number reindex_rethrottle@200: - description: '' content: application/json: schema: @@ -3044,7 +3011,6 @@ components: required: - nodes render_search_template@200: - description: '' content: application/json: schema: @@ -3057,7 +3023,6 @@ components: required: - template_output scripts_painless_execute@200: - description: '' content: application/json: schema: @@ -3068,19 +3033,16 @@ components: required: - result scroll@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.search.yaml#/components/schemas/ResponseBody' search@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.search.yaml#/components/schemas/ResponseBody' search_shards@200: - description: '' content: application/json: schema: @@ -3105,7 +3067,6 @@ components: - nodes - shards search_template@200: - description: '' content: application/json: schema: @@ -3153,7 +3114,6 @@ components: - timed_out - took termvectors@200: - description: '' content: application/json: schema: @@ -3180,13 +3140,11 @@ components: - found - took update@200: - description: '' content: application/json: schema: $ref: '../schemas/_core.update.yaml#/components/schemas/UpdateWriteResponseBase' update_by_query@200: - description: '' content: application/json: schema: @@ -3227,7 +3185,6 @@ components: throttled_until_millis: $ref: '../schemas/_common.yaml#/components/schemas/DurationValueUnitMillis' update_by_query_rethrottle@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index 76a2d8b80..b22637624 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Cat API - description: OpenSearch Cat API + description: OpenSearch Cat API. version: 1.0.0 paths: /_cat: @@ -787,7 +787,6 @@ components: - pit_id responses: cat.aliases@200: - description: '' content: text/plain: type: string @@ -797,7 +796,6 @@ components: items: $ref: '../schemas/cat.aliases.yaml#/components/schemas/AliasesRecord' cat.all_pit_segments@200: - description: '' content: text/plain: type: string @@ -807,7 +805,6 @@ components: items: $ref: '../schemas/cat._common.yaml#/components/schemas/CatPitSegmentsRecord' cat.allocation@200: - description: '' content: text/plain: type: string @@ -817,7 +814,6 @@ components: items: $ref: '../schemas/cat.allocation.yaml#/components/schemas/AllocationRecord' cat.cluster_manager@200: - description: '' content: text/plain: type: string @@ -827,7 +823,6 @@ components: items: $ref: '../schemas/cat.cluster_manager.yaml#/components/schemas/ClusterManagerRecord' cat.count@200: - description: '' content: text/plain: type: string @@ -837,7 +832,6 @@ components: items: $ref: '../schemas/cat.count.yaml#/components/schemas/CountRecord' cat.fielddata@200: - description: '' content: text/plain: type: string @@ -847,7 +841,6 @@ components: items: $ref: '../schemas/cat.fielddata.yaml#/components/schemas/FielddataRecord' cat.health@200: - description: '' content: text/plain: type: string @@ -872,13 +865,11 @@ components: items: $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord' cat.help@200: - description: '' content: text/plain: schema: type: string cat.indices@200: - description: '' content: text/plain: type: string @@ -903,7 +894,6 @@ components: items: $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord' cat.master@200: - description: '' content: text/plain: type: string @@ -913,7 +903,6 @@ components: items: $ref: '../schemas/cat.master.yaml#/components/schemas/MasterRecord' cat.nodeattrs@200: - description: '' content: text/plain: type: string @@ -923,7 +912,6 @@ components: items: $ref: '../schemas/cat.nodeattrs.yaml#/components/schemas/NodeAttributesRecord' cat.nodes@200: - description: '' content: text/plain: type: string @@ -933,7 +921,6 @@ components: items: $ref: '../schemas/cat.nodes.yaml#/components/schemas/NodesRecord' cat.pending_tasks@200: - description: '' content: text/plain: type: string @@ -943,7 +930,6 @@ components: items: $ref: '../schemas/cat.pending_tasks.yaml#/components/schemas/PendingTasksRecord' cat.pit_segments@200: - description: '' content: text/plain: type: string @@ -953,7 +939,6 @@ components: items: $ref: '../schemas/cat._common.yaml#/components/schemas/CatPitSegmentsRecord' cat.plugins@200: - description: '' content: text/plain: type: string @@ -963,7 +948,6 @@ components: items: $ref: '../schemas/cat.plugins.yaml#/components/schemas/PluginsRecord' cat.recovery@200: - description: '' content: text/plain: type: string @@ -973,7 +957,6 @@ components: items: $ref: '../schemas/cat.recovery.yaml#/components/schemas/RecoveryRecord' cat.repositories@200: - description: '' content: text/plain: type: string @@ -983,7 +966,6 @@ components: items: $ref: '../schemas/cat.repositories.yaml#/components/schemas/RepositoriesRecord' cat.segment_replication@200: - description: '' content: text/plain: type: string @@ -993,7 +975,6 @@ components: items: $ref: '../schemas/cat._common.yaml#/components/schemas/CatSegmentReplicationRecord' cat.segments@200: - description: '' content: text/plain: type: string @@ -1003,7 +984,6 @@ components: items: $ref: '../schemas/cat.segments.yaml#/components/schemas/SegmentsRecord' cat.shards@200: - description: '' content: text/plain: type: string @@ -1013,7 +993,6 @@ components: items: $ref: '../schemas/cat.shards.yaml#/components/schemas/ShardsRecord' cat.snapshots@200: - description: '' content: text/plain: type: string @@ -1023,7 +1002,6 @@ components: items: $ref: '../schemas/cat.snapshots.yaml#/components/schemas/SnapshotsRecord' cat.tasks@200: - description: '' content: text/plain: type: string @@ -1033,7 +1011,6 @@ components: items: $ref: '../schemas/cat.tasks.yaml#/components/schemas/TasksRecord' cat.templates@200: - description: '' content: text/plain: type: string @@ -1043,7 +1020,6 @@ components: items: $ref: '../schemas/cat.templates.yaml#/components/schemas/TemplatesRecord' cat.thread_pool@200: - description: '' content: text/plain: type: string diff --git a/spec/namespaces/cluster.yaml b/spec/namespaces/cluster.yaml index d0b8c5fd2..c503c375b 100644 --- a/spec/namespaces/cluster.yaml +++ b/spec/namespaces/cluster.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Cluster API - description: OpenSearch Cluster API + description: OpenSearch Cluster API. version: 1.0.0 paths: /_cluster/allocation/explain: @@ -534,7 +534,6 @@ components: description: The definition of `commands` to perform (`move`, `cancel`, `allocate`) responses: cluster.allocation_explain@200: - description: '' content: application/json: schema: @@ -602,25 +601,19 @@ components: - primary - shard cluster.delete_component_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' - cluster.delete_decommission_awareness@200: - description: '' + cluster.delete_decommission_awareness@200: {} cluster.delete_voting_config_exclusions@200: - description: '' content: application/json: {} - cluster.delete_weighted_routing@200: - description: '' + cluster.delete_weighted_routing@200: {} cluster.exists_component_template@200: - description: '' content: application/json: {} cluster.get_component_template@200: - description: '' content: application/json: schema: @@ -632,10 +625,8 @@ components: $ref: '../schemas/cluster._common.yaml#/components/schemas/ComponentTemplate' required: - component_templates - cluster.get_decommission_awareness@200: - description: '' + cluster.get_decommission_awareness@200: {} cluster.get_settings@200: - description: '' content: application/json: schema: @@ -656,16 +647,13 @@ components: required: - persistent - transient - cluster.get_weighted_routing@200: - description: '' + cluster.get_weighted_routing@200: {} cluster.health@200: - description: '' content: application/json: schema: $ref: '../schemas/cluster.health.yaml#/components/schemas/HealthResponseBody' cluster.pending_tasks@200: - description: '' content: application/json: schema: @@ -678,19 +666,15 @@ components: required: - tasks cluster.post_voting_config_exclusions@200: - description: '' content: application/json: {} cluster.put_component_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' - cluster.put_decommission_awareness@200: - description: '' + cluster.put_decommission_awareness@200: {} cluster.put_settings@200: - description: '' content: application/json: schema: @@ -708,10 +692,8 @@ components: - acknowledged - persistent - transient - cluster.put_weighted_routing@200: - description: '' + cluster.put_weighted_routing@200: {} cluster.remote_info@200: - description: '' content: application/json: schema: @@ -719,7 +701,6 @@ components: additionalProperties: $ref: '../schemas/cluster.remote_info.yaml#/components/schemas/ClusterRemoteInfo' cluster.reroute@200: - description: '' content: application/json: schema: @@ -740,13 +721,11 @@ components: required: - acknowledged cluster.state@200: - description: '' content: application/json: schema: type: object cluster.stats@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/dangling_indices.yaml b/spec/namespaces/dangling_indices.yaml index 341757ad8..73b8763cb 100644 --- a/spec/namespaces/dangling_indices.yaml +++ b/spec/namespaces/dangling_indices.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Dangling_indices API - description: OpenSearch Dangling_indices API + description: OpenSearch Dangling_indices API. version: 1.0.0 paths: /_dangling: @@ -53,19 +53,16 @@ components: requestBodies: {} responses: dangling_indices.delete_dangling_index@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' dangling_indices.import_dangling_index@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' dangling_indices.list_dangling_indices@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index 3eca32255..6d36982f5 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Indices API - description: OpenSearch Indices API + description: OpenSearch Indices API. version: 1.0.0 paths: /_alias: @@ -2249,7 +2249,6 @@ components: description: The query definition specified with the Query DSL responses: indices.add_block@200: - description: '' content: application/json: schema: @@ -2268,7 +2267,6 @@ components: - indices - shards_acknowledged indices.analyze@200: - description: '' content: application/json: schema: @@ -2281,13 +2279,11 @@ components: items: $ref: '../schemas/indices.analyze.yaml#/components/schemas/AnalyzeToken' indices.clear_cache@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/ShardsOperationResponseBase' indices.clone@200: - description: '' content: application/json: schema: @@ -2304,7 +2300,6 @@ components: - index - shards_acknowledged indices.close@200: - description: '' content: application/json: schema: @@ -2323,7 +2318,6 @@ components: - indices - shards_acknowledged indices.create@200: - description: '' content: application/json: schema: @@ -2340,13 +2334,11 @@ components: - index - shards_acknowledged indices.create_data_stream@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.data_streams_stats@200: - description: '' content: application/json: schema: @@ -2377,63 +2369,51 @@ components: - data_streams - total_store_size_bytes indices.delete@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndicesResponseBase' indices.delete_alias@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.delete_data_stream@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.delete_index_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.delete_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.exists@200: - description: '' content: application/json: {} indices.exists@404: - description: '' content: application/json: {} indices.exists_alias@200: - description: '' content: application/json: {} indices.exists_index_template@200: - description: '' content: application/json: {} indices.exists_template@200: - description: '' content: application/json: {} indices.flush@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/ShardsOperationResponseBase' indices.forcemerge@200: - description: '' content: application/json: schema: @@ -2447,7 +2427,6 @@ components: you can use the task_id to get the status of the task at _tasks/ type: string indices.get@200: - description: '' content: application/json: schema: @@ -2455,7 +2434,6 @@ components: additionalProperties: $ref: '../schemas/indices._common.yaml#/components/schemas/IndexState' indices.get_alias@200: - description: '' content: application/json: schema: @@ -2463,7 +2441,6 @@ components: additionalProperties: $ref: '../schemas/indices.get_alias.yaml#/components/schemas/IndexAliases' indices.get_data_stream@200: - description: '' content: application/json: schema: @@ -2476,7 +2453,6 @@ components: required: - data_streams indices.get_field_mapping@200: - description: '' content: application/json: schema: @@ -2484,7 +2460,6 @@ components: additionalProperties: $ref: '../schemas/indices.get_field_mapping.yaml#/components/schemas/TypeFieldMappings' indices.get_index_template@200: - description: '' content: application/json: schema: @@ -2497,7 +2472,6 @@ components: required: - index_templates indices.get_mapping@200: - description: '' content: application/json: schema: @@ -2505,7 +2479,6 @@ components: additionalProperties: $ref: '../schemas/indices.get_mapping.yaml#/components/schemas/IndexMappingRecord' indices.get_settings@200: - description: '' content: application/json: schema: @@ -2513,17 +2486,14 @@ components: additionalProperties: $ref: '../schemas/indices._common.yaml#/components/schemas/IndexState' indices.get_template@200: - description: '' content: application/json: schema: type: object additionalProperties: $ref: '../schemas/indices._common.yaml#/components/schemas/TemplateMapping' - indices.get_upgrade@200: - description: '' + indices.get_upgrade@200: {} indices.open@200: - description: '' content: application/json: schema: @@ -2544,37 +2514,31 @@ components: - shards_acknowledged additionalProperties: false indices.put_alias@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.put_index_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.put_mapping@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/IndicesResponseBase' indices.put_settings@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.put_template@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' indices.recovery@200: - description: '' content: application/json: schema: @@ -2582,13 +2546,11 @@ components: additionalProperties: $ref: '../schemas/indices.recovery.yaml#/components/schemas/RecoveryStatus' indices.refresh@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/ShardsOperationResponseBase' indices.resolve_index@200: - description: '' content: application/json: schema: @@ -2611,7 +2573,6 @@ components: - data_streams - indices indices.rollover@200: - description: '' content: application/json: schema: @@ -2642,7 +2603,6 @@ components: - rolled_over - shards_acknowledged indices.segments@200: - description: '' content: application/json: schema: @@ -2658,7 +2618,6 @@ components: - _shards - indices indices.shard_stores@200: - description: '' content: application/json: schema: @@ -2671,7 +2630,6 @@ components: required: - indices indices.shrink@200: - description: '' content: application/json: schema: @@ -2688,13 +2646,11 @@ components: - index - shards_acknowledged indices.simulate_index_template@200: - description: '' content: application/json: schema: type: object indices.simulate_template@200: - description: '' content: application/json: schema: @@ -2709,7 +2665,6 @@ components: required: - template indices.split@200: - description: '' content: application/json: schema: @@ -2726,7 +2681,6 @@ components: - index - shards_acknowledged indices.stats@200: - description: '' content: application/json: schema: @@ -2744,15 +2698,12 @@ components: - _all - _shards indices.update_aliases@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' - indices.upgrade@200: - description: '' + indices.upgrade@200: {} indices.validate_query@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/ingest.yaml b/spec/namespaces/ingest.yaml index fa21ca569..787d1d509 100644 --- a/spec/namespaces/ingest.yaml +++ b/spec/namespaces/ingest.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Ingest API - description: OpenSearch Ingest API + description: OpenSearch Ingest API. version: 1.0.0 paths: /_ingest/pipeline: @@ -181,13 +181,11 @@ components: required: true responses: ingest.delete_pipeline@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' ingest.get_pipeline@200: - description: '' content: application/json: schema: @@ -195,7 +193,6 @@ components: additionalProperties: $ref: '../schemas/ingest._common.yaml#/components/schemas/Pipeline' ingest.processor_grok@200: - description: '' content: application/json: schema: @@ -208,13 +205,11 @@ components: required: - patterns ingest.put_pipeline@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' ingest.simulate@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/knn.yaml b/spec/namespaces/knn.yaml index 0ef35ae6f..d960f34a5 100644 --- a/spec/namespaces/knn.yaml +++ b/spec/namespaces/knn.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Knn API - description: OpenSearch Knn API + description: OpenSearch Knn API. version: 1.0.0 paths: /_plugins/_knn/models/_search: @@ -277,18 +277,12 @@ components: - training_index required: true responses: - knn.delete_model@200: - description: '' - knn.get_model@200: - description: '' - knn.search_models@200: - description: '' - knn.stats@200: - description: '' - knn.train_model@200: - description: '' - knn.warmup@200: - description: '' + knn.delete_model@200: {} + knn.get_model@200: {} + knn.search_models@200: {} + knn.stats@200: {} + knn.train_model@200: {} + knn.warmup@200: {} parameters: knn.delete_model::path.model_id: name: model_id diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index c60605851..54b5ea532 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch ML API - description: OpenSearch ML API + description: OpenSearch ML API. version: 1.0.0 paths: /_plugins/_ml/model_groups/_register: diff --git a/spec/namespaces/nodes.yaml b/spec/namespaces/nodes.yaml index e99653b12..e8c919ebc 100644 --- a/spec/namespaces/nodes.yaml +++ b/spec/namespaces/nodes.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Nodes API - description: OpenSearch Nodes API + description: OpenSearch Nodes API. version: 1.0.0 paths: /_cluster/nodes/hot_threads: @@ -449,28 +449,23 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/Password' description: An object containing the password for the opensearch keystore responses: - nodes.hot_threads@200: - description: '' + nodes.hot_threads@200: {} nodes.info@200: - description: '' content: application/json: schema: $ref: '../schemas/nodes.info.yaml#/components/schemas/ResponseBase' nodes.reload_secure_settings@200: - description: '' content: application/json: schema: $ref: '../schemas/nodes.reload_secure_settings.yaml#/components/schemas/ResponseBase' nodes.stats@200: - description: '' content: application/json: schema: $ref: '../schemas/nodes.stats.yaml#/components/schemas/ResponseBase' nodes.usage@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/notifications.yaml b/spec/namespaces/notifications.yaml index edb7ef077..00faf3941 100644 --- a/spec/namespaces/notifications.yaml +++ b/spec/namespaces/notifications.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Notifications API - description: OpenSearch Notifications API + description: OpenSearch Notifications API. version: 1.0.0 paths: /_plugins/_notifications/channels: @@ -204,7 +204,6 @@ components: required: true responses: notifications.create_config@200: - description: '' content: application/json: schema: @@ -213,31 +212,26 @@ components: config_id: type: string notifications.delete_config@200: - description: '' content: application/json: schema: $ref: '../schemas/notifications._common.yaml#/components/schemas/DeleteConfigsResponse' notifications.delete_configs@200: - description: '' content: application/json: schema: $ref: '../schemas/notifications._common.yaml#/components/schemas/DeleteConfigsResponse' notifications.get_config@200: - description: '' content: application/json: schema: $ref: '../schemas/notifications._common.yaml#/components/schemas/GetConfigsResponse' notifications.get_configs@200: - description: '' content: application/json: schema: $ref: '../schemas/notifications._common.yaml#/components/schemas/GetConfigsResponse' notifications.list_channels@200: - description: '' content: application/json: schema: @@ -256,7 +250,6 @@ components: items: $ref: '../schemas/notifications._common.yaml#/components/schemas/NotificationChannel' notifications.list_features@200: - description: '' content: application/json: schema: @@ -269,7 +262,6 @@ components: plugin_features: $ref: '../schemas/notifications._common.yaml#/components/schemas/NotificationsPluginFeaturesMap' notifications.send_test@200: - description: '' content: application/json: schema: @@ -282,7 +274,6 @@ components: items: $ref: '../schemas/notifications._common.yaml#/components/schemas/EventStatus' notifications.update_config@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/ppl.yaml b/spec/namespaces/ppl.yaml index 7515cea94..0794fd32d 100644 --- a/spec/namespaces/ppl.yaml +++ b/spec/namespaces/ppl.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch PPL API - description: OpenSearch PPL API + description: OpenSearch PPL API. version: 1.0.0 paths: /_plugins/_ppl: diff --git a/spec/namespaces/remote_store.yaml b/spec/namespaces/remote_store.yaml index a1f1ffb2d..4202d5cdf 100644 --- a/spec/namespaces/remote_store.yaml +++ b/spec/namespaces/remote_store.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Remote_store API - description: OpenSearch Remote_store API + description: OpenSearch Remote_store API. version: 1.0.0 paths: /_remotestore/_restore: @@ -38,14 +38,12 @@ components: examples: RemoteStoreRestore_example1: summary: Examples for Post Remote Storage Restore Operation. - description: '' value: indices: - books required: true responses: remote_store.restore@200: - description: '' content: application/json: schema: @@ -58,7 +56,6 @@ components: examples: RemoteStoreRestore_example1: summary: Examples for Post Remote Storage Restore Operation. - description: '' value: remote_store: indices: diff --git a/spec/namespaces/rollups.yaml b/spec/namespaces/rollups.yaml index a737139ed..e5e56a68a 100644 --- a/spec/namespaces/rollups.yaml +++ b/spec/namespaces/rollups.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Index Rollups API - description: OpenSearch Index Rollups API + description: OpenSearch Index Rollups API. version: 1.0.0 paths: /_plugins/_rollup/jobs/{id}: @@ -102,12 +102,9 @@ components: application/json: schema: $ref: '../schemas/rollups._common.yaml#/components/schemas/RollupEntity' - rollups.delete@200: - description: '' - rollups.start@200: - description: '' - rollups.stop@200: - description: '' + rollups.delete@200: {} + rollups.start@200: {} + rollups.stop@200: {} rollups.explain@200: content: application/json: diff --git a/spec/namespaces/search_pipeline.yaml b/spec/namespaces/search_pipeline.yaml index 3ce4c5d80..05d16fbdd 100644 --- a/spec/namespaces/search_pipeline.yaml +++ b/spec/namespaces/search_pipeline.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Search_pipeline API - description: OpenSearch Search_pipeline API + description: OpenSearch Search_pipeline API. version: 1.0.0 paths: /_search/pipeline: @@ -65,7 +65,6 @@ components: required: true responses: search_pipeline.put@200: - description: '' content: application/json: schema: @@ -74,7 +73,6 @@ components: acknowledged: type: boolean search_pipeline.delete@200: - description: '' content: application/json: schema: @@ -83,7 +81,6 @@ components: acknowledged: type: boolean search_pipeline.get@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 234ab6fc7..f41647f59 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Security API - description: OpenSearch Security API + description: OpenSearch Security API. version: 1.0.0 paths: /_opendistro/_security/sslinfo: @@ -1228,259 +1228,216 @@ components: $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' responses: security.authinfo@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AuthInfo' security.authinfo@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.authtoken@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.cache@501: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/MethodNotImplemented' security.change_password@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.config_upgrade_check@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UpgradeCheck' security.config_upgrade_perform@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UpgradePerform' security.create_action_group@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.create_allowlist@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.create_role@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.create_role_mapping@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.create_tenant@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.create_tenant@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.create_update_tenancy_config@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' security.create_update_tenancy_config@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.create_user@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.create_user_legacy@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_action_group@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_distinguished_name@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_distinguished_name@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.delete_role@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_role_mapping@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_tenant@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_tenant@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.delete_user@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.delete_user_legacy@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.flush_cache@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.generate_obo_token@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/GenerateOBOToken' security.generate_obo_token@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.generate_user_token@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.generate_user_token@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.generate_user_token_legacy@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.generate_user_token_legacy@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.get_account_details@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AccountDetails' security.get_action_group@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/ActionGroupsMap' security.get_action_groups@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/ActionGroupsMap' security.get_allowlist@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.get_audit_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AuditConfigWithReadOnly' security.get_certificates@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificates' security.get_certificates@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.get_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DynamicConfig' security.get_dashboards_info@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DashboardsInfo' security.get_dashboards_info@500: - description: '' content: application/json: schema: @@ -1492,385 +1449,321 @@ components: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' security.get_distinguished_name@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.get_distinguished_names@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNamesMap' security.get_distinguished_names@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.get_permissions_info@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/PermissionsInfo' security.get_permissions_info@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.get_role@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/RolesMap' security.get_role_mapping@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/RoleMappings' security.get_role_mappings@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/RoleMappings' security.get_roles@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/RolesMap' security.get_sslinfo@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/SSLInfo' security.get_sslinfo@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.get_tenant@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/TenantsMap' security.get_tenancy_config@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' security.get_tenancy_config@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.get_tenants@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/TenantsMap' security.get_tenants@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.get_user@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UsersMap' security.get_user_legacy@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UsersMap' security.get_users@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UsersMap' security.get_users_legacy@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UsersMap' security.health@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/HealthInfo' security.migrate@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.migrate@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_action_group@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_action_groups@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_allowlist@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' security.patch_audit_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_distinguished_name@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_distinguished_name@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.patch_distinguished_names@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_distinguished_names@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.patch_role@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_role@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_role_mapping@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_role_mapping@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_role_mappings@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_role_mappings@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_roles@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_roles@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_tenant@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_tenant@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_tenants@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_tenants@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.patch_user@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.patch_users@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.post_dashboards_info@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DashboardsInfo' security.post_dashboards_info@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.reload_http_certificates@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.reload_http_certificates@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.reload_transport_certificates@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.reload_transport_certificates@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.tenant_info@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/TenantInfo' security.tenant_info@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.update_audit_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.update_configuration@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.update_distinguished_name@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.update_distinguished_name@403: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' security.validate@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.validate@400: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' security.who_am_i@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/WhoAmI' security.who_am_i@500: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.who_am_i_protected@200: - description: '' content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/WhoAmI' security.who_am_i_protected@500: - description: '' content: application/json: schema: diff --git a/spec/namespaces/snapshot.yaml b/spec/namespaces/snapshot.yaml index 3847af1db..9f6c88c00 100644 --- a/spec/namespaces/snapshot.yaml +++ b/spec/namespaces/snapshot.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Snapshot API - description: OpenSearch Snapshot API + description: OpenSearch Snapshot API. version: 1.0.0 paths: /_snapshot: @@ -362,7 +362,6 @@ components: description: Details of what to restore responses: snapshot.cleanup_repository@200: - description: '' content: application/json: schema: @@ -373,13 +372,11 @@ components: required: - results snapshot.clone@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' snapshot.create@200: - description: '' content: application/json: schema: @@ -391,25 +388,21 @@ components: snapshot: $ref: '../schemas/snapshot._common.yaml#/components/schemas/SnapshotInfo' snapshot.create_repository@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' snapshot.delete@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' snapshot.delete_repository@200: - description: '' content: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' snapshot.get@200: - description: '' content: application/json: schema: @@ -433,7 +426,6 @@ components: - remaining - total snapshot.get_repository@200: - description: '' content: application/json: schema: @@ -441,7 +433,6 @@ components: additionalProperties: $ref: '../schemas/snapshot._common.yaml#/components/schemas/Repository' snapshot.restore@200: - description: '' content: application/json: schema: @@ -452,7 +443,6 @@ components: required: - snapshot snapshot.status@200: - description: '' content: application/json: schema: @@ -465,7 +455,6 @@ components: required: - snapshots snapshot.verify_repository@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/sql.yaml b/spec/namespaces/sql.yaml index 7de1b1c22..a54c6cc39 100644 --- a/spec/namespaces/sql.yaml +++ b/spec/namespaces/sql.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch SQL API - description: OpenSearch SQL API + description: OpenSearch SQL API. version: 1.0.0 paths: /_plugins/_query/settings: diff --git a/spec/namespaces/tasks.yaml b/spec/namespaces/tasks.yaml index 9f5bf8dc5..013155f8f 100644 --- a/spec/namespaces/tasks.yaml +++ b/spec/namespaces/tasks.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Tasks API - description: OpenSearch Tasks API + description: OpenSearch Tasks API. version: 1.0.0 paths: /_tasks: @@ -75,13 +75,11 @@ components: requestBodies: {} responses: tasks.cancel@200: - description: '' content: application/json: schema: $ref: '../schemas/tasks._common.yaml#/components/schemas/TaskListResponseBase' tasks.get@200: - description: '' content: application/json: schema: @@ -99,7 +97,6 @@ components: - completed - task tasks.list@200: - description: '' content: application/json: schema: diff --git a/spec/namespaces/transforms.yaml b/spec/namespaces/transforms.yaml index 336cd18af..b525f101a 100644 --- a/spec/namespaces/transforms.yaml +++ b/spec/namespaces/transforms.yaml @@ -1,7 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Index Transforms API - description: OpenSearch Index Transforms API + description: OpenSearch Index Transforms API. version: 1.0.0 paths: /_plugins/_transform: @@ -80,7 +80,7 @@ paths: operationId: transforms.stop.0 x-operation-group: transforms.stop x-version-added: '1.0' - description: stop transform. + description: Stop transform. externalDocs: url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#stop-a-transform-job parameters: @@ -135,8 +135,7 @@ components: application/json: schema: $ref: '../schemas/transforms._common.yaml#/components/schemas/TransformEntity' - transforms.delete@200: - description: '' + transforms.delete@200: {} transforms.start@200: content: application/json: diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 414272b8b..a0d547cd5 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -1637,7 +1637,7 @@ components: required: - query SimpleQueryStringFlags: - description: Query flags can be either a single flag or a combination of flags, e.g. `OR|AND|PREFIX` + description: Query flags can be either a single flag or a combination of flags, e.g. `OR|AND|PREFIX`. allOf: - $ref: '_common.yaml#/components/schemas/PipeSeparatedFlagsSimpleQueryStringFlag' SimpleQueryStringFlag: diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 81c69f230..afb4b6d1a 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -49,7 +49,7 @@ components: - as 4 top/bottom/left/right coordinates - as 2 top_left / bottom_right points - as 2 top_right / bottom_left points - - as a WKT bounding box + - as a WKT bounding box. oneOf: - $ref: '#/components/schemas/CoordsGeoBounds' - $ref: '#/components/schemas/TopLeftBottomRightGeoBounds' @@ -87,7 +87,7 @@ components: - as a `{lat, long}` object - as a geo hash value - as a `[lon, lat]` array - - as a string in `", "` or WKT point formats + - as a string in `", "` or WKT point formats. oneOf: - $ref: '#/components/schemas/LatLonGeoLocation' - $ref: '#/components/schemas/GeoHashLocation' @@ -137,12 +137,11 @@ components: allOf: - $ref: '#/components/schemas/UnitMillis' UnitMillis: - description: Time unit for milliseconds + description: Time unit for milliseconds. type: number DurationLarge: description: |- - A date histogram interval. Similar to `Duration` with additional units: `w` (week), `M` (month), `q` (quarter) and - `y` (year) + A date histogram interval. Similar to `Duration` with additional units: `w` (week), `M` (month), `q` (quarter) and `y` (year). type: string FieldValue: description: A field value. @@ -159,13 +158,13 @@ components: Although "void" is generally used for the unit type that has only one value, this is to be interpreted as the bottom type that has no value at all. Most languages have a unit type, but few have a bottom type. - See https://en.m.wikipedia.org/wiki/Unit_type and https://en.m.wikipedia.org/wiki/Bottom_type + See https://en.m.wikipedia.org/wiki/Unit_type and https://en.m.wikipedia.org/wiki/Bottom_type. type: object GeoTile: - description: A map tile reference, represented as `{zoom}/{x}/{y}` + description: A map tile reference, represented as `{zoom}/{x}/{y}`. type: string GeoHexCell: - description: A map hex cell (H3) reference + description: A map hex cell (H3) reference. type: string IndexName: type: string @@ -319,7 +318,7 @@ components: allOf: - $ref: '#/components/schemas/UnitNanos' UnitNanos: - description: Time unit for nanoseconds + description: Time unit for nanoseconds. type: number ScrollId: type: string @@ -373,7 +372,7 @@ components: - missing - popular MinimumShouldMatch: - description: The minimum number of terms that should match as integer, percentage or range + description: The minimum number of terms that should match as integer, percentage or range. oneOf: - type: number - type: string @@ -742,10 +741,10 @@ components: allOf: - $ref: '#/components/schemas/UnitSeconds' UnitSeconds: - description: Time unit for seconds + description: Time unit for seconds. type: number TimeOfDay: - description: Time of day, expressed as HH:MM:SS + description: Time of day, expressed as HH:MM:SS. type: string TimeUnit: type: string @@ -1907,7 +1906,7 @@ components: - name - transport_address NodeRoles: - description: '* @doc_id node-roles' + description: Node roles. type: array items: $ref: '#/components/schemas/NodeRole' diff --git a/spec/schemas/indices._common.yaml b/spec/schemas/indices._common.yaml index 8cf004a9c..fce77a57d 100644 --- a/spec/schemas/indices._common.yaml +++ b/spec/schemas/indices._common.yaml @@ -8,6 +8,7 @@ components: schemas: IndexSettings: type: object + description: The index settings to be updated. properties: index: $ref: '#/components/schemas/IndexSettings' @@ -138,7 +139,6 @@ components: store: $ref: '#/components/schemas/Storage' additionalProperties: {} - description: The index settings to be updated SoftDeletes: type: object properties: @@ -953,7 +953,7 @@ components: $ref: '#/components/schemas/IndexTemplateDataStreamConfiguration' required: - index_patterns - description: New index template definition to be simulated, if no index template name is specified + description: New index template definition to be simulated, if no index template name is specified. IndexTemplateSummary: type: object properties: diff --git a/tools/src/linter/components/Info.ts b/tools/src/linter/components/Info.ts new file mode 100644 index 000000000..0ec39beca --- /dev/null +++ b/tools/src/linter/components/Info.ts @@ -0,0 +1,47 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { type OpenAPIV3 } from 'openapi-types' +import { type ValidationError } from 'types' +import ValidatorBase from './base/ValidatorBase' +import { toLaxTitleCase } from 'titlecase' + +const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u + +export default class Info extends ValidatorBase { + path: string + spec: OpenAPIV3.InfoObject + + constructor (file: string, path: string, spec: OpenAPIV3.InfoObject) { + super(file, 'Info') + this.path = path + this.spec = spec + } + + validate (): ValidationError[] { + return [ + this.validate_title(), + this.validate_description() + ].filter((e) => e) as ValidationError[] + } + + validate_description (): ValidationError | undefined { + const description = this.spec?.description ?? '' + if (description === '') { return this.error('Missing description property.') } + if (!DESCRIPTION_REGEX.test(description)) { return this.error('Description must start with a capital letter and end with a period.') } + } + + validate_title (): ValidationError | undefined { + const title = this.spec?.title ?? '' + if (title === '') { return this.error('Missing title property.') } + const expected_title = toLaxTitleCase(title) + if (title.endsWith('.')) { return this.error('Title must not end with a period.') } + if (title !== expected_title) return this.error(`Title must be capitalized, expected '${expected_title}'.`) + } +} diff --git a/tools/src/linter/components/NamespaceFile.ts b/tools/src/linter/components/NamespaceFile.ts index fdfb2d0ef..7b738e16f 100644 --- a/tools/src/linter/components/NamespaceFile.ts +++ b/tools/src/linter/components/NamespaceFile.ts @@ -14,12 +14,14 @@ import _ from 'lodash' import Operation from './Operation' import { resolve_ref, sort_by_keys } from '../../helpers' import FileValidator from './base/FileValidator' +import Info from './Info' const HTTP_METHODS = ['get', 'head', 'post', 'put', 'patch', 'delete', 'options', 'trace'] const NAME_REGEX = /^[a-z]+[a-z_]*[a-z]+$/ export default class NamespaceFile extends FileValidator { namespace: string + private _info: Info | undefined private _operation_groups: OperationGroup[] | undefined private _refs: Set | undefined @@ -36,6 +38,7 @@ export default class NamespaceFile extends FileValidator { return [ this.validate_schemas(), + ...this.validate_info(), ...this.validate_unresolved_refs(), ...this.validate_unused_refs(), ...this.validate_parameter_refs(), @@ -43,6 +46,12 @@ export default class NamespaceFile extends FileValidator { ].filter((e) => e) as ValidationError[] } + info (): Info { + if (this._info) return this._info + this._info = new Info(this.file, this.file_path, this.spec().info) + return this._info + } + operation_groups (): OperationGroup[] { if (this._operation_groups) return this._operation_groups const ops: Operation[] = _.entries(this.spec().paths).flatMap(([path, ops]) => { @@ -78,6 +87,10 @@ export default class NamespaceFile extends FileValidator { if (this.spec().components?.schemas) { return this.error('components/schemas is not allowed in namespace files', '#/components/schemas') } } + validate_info(): ValidationError[] { + return this.info().validate() + } + validate_unresolved_refs (): ValidationError[] { return Array.from(this.refs()).map((ref) => { if (resolve_ref(ref, this.spec()) === undefined) return this.error(`Unresolved reference: ${ref}`, ref) diff --git a/tools/src/linter/components/Operation.ts b/tools/src/linter/components/Operation.ts index 742b0208c..bb06233ad 100644 --- a/tools/src/linter/components/Operation.ts +++ b/tools/src/linter/components/Operation.ts @@ -12,6 +12,8 @@ import _ from 'lodash' import ValidatorBase from './base/ValidatorBase' const GROUP_REGEX = /^([a-z]+[a-z_]*[a-z]+\.)?([a-z]+[a-z_]*[a-z]+)$/ +const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u + export default class Operation extends ValidatorBase { path: string verb: string @@ -65,7 +67,7 @@ export default class Operation extends ValidatorBase { validate_description (): ValidationError | undefined { const description = this.spec.description ?? '' if (description === '') { return this.error('Missing description property.') } - if (!description.endsWith('.')) { return this.error('Description must end with a period.') } + if (!DESCRIPTION_REGEX.test(description)) return this.error('Description must start with a capital letter and end with a period.') } validate_operation_id (): ValidationError | undefined { diff --git a/tools/src/linter/components/Schema.ts b/tools/src/linter/components/Schema.ts index 9178950a0..a84438f92 100644 --- a/tools/src/linter/components/Schema.ts +++ b/tools/src/linter/components/Schema.ts @@ -12,6 +12,7 @@ import { type OpenAPIV3 } from 'openapi-types' import { type ValidationError } from 'types' const NAME_REGEX = /^[A-Za-z0-9]+$/ +const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u export default class Schema extends ValidatorBase { name: string @@ -24,10 +25,19 @@ export default class Schema extends ValidatorBase { } validate (): ValidationError[] { - return [this.validate_name()].filter(e => e) as ValidationError[] + return [ + this.validate_name(), + this.validate_description() + ].filter(e => e) as ValidationError[] } validate_name (): ValidationError | undefined { if (!NAME_REGEX.test(this.name)) { return this.error(`Invalid schema name '${this.name}'. Only alphanumeric characters are allowed.`) } } + + validate_description (): ValidationError | undefined { + const description = this.spec.description ?? '' + if (description === '') return + if (!DESCRIPTION_REGEX.test(description)) { return this.error('Description must start with a capital letter and end with a period.') } + } } diff --git a/tools/tests/linter/NamespaceFile.test.ts b/tools/tests/linter/NamespaceFile.test.ts index 6e5ff9eb3..88cb22c91 100644 --- a/tools/tests/linter/NamespaceFile.test.ts +++ b/tools/tests/linter/NamespaceFile.test.ts @@ -99,23 +99,69 @@ test('validate_order_of_operations()', () => { ]) }) +test('validate_info() periods', () => { + const validator = namespace_file('invalid_info_periods.yaml') + expect(validator.info().validate()).toEqual([ + { + file: 'namespaces/invalid_info_periods.yaml', + location: 'Info', + message: 'Title must not end with a period.' + }, + { + file: 'namespaces/invalid_info_periods.yaml', + location: 'Info', + message: 'Description must start with a capital letter and end with a period.' + } + ]) +}) + +test('validate_info() multiline', () => { + const validator = namespace_file('valid_info_multiline_description.yaml') + expect(validator.info().validate()).toEqual([]) +}) + +test('validate_info() capitals', () => { + const validator = namespace_file('invalid_info_capitals.yaml') + expect(validator.info().validate()).toEqual([ + { + file: 'namespaces/invalid_info_capitals.yaml', + location: 'Info', + message: "Title must be capitalized, expected 'Title Must Be Capitalized'." + }, + { + file: 'namespaces/invalid_info_capitals.yaml', + location: 'Info', + message: "Description must start with a capital letter and end with a period." + } + ]) +}) + test('validate()', () => { - const invalid_name = mocked_namespace_file({ returned_values: { validate_name: 'Invalid Name' }, groups_errors: [['group error']] }) + const invalid_name = mocked_namespace_file({ returned_values: { validate_name: 'Invalid Name' } }) expect(invalid_name.validate()).toEqual(['Invalid Name']) - const invalid_groups = mocked_namespace_file({ returned_values: { validate_schemas: 'Invalid schemas' }, groups_errors: [['error']] }) + const invalid_groups = mocked_namespace_file({ groups_errors: [['error']] }) expect(invalid_groups.validate()).toEqual(['error']) const typical = mocked_namespace_file({ returned_values: { + validate_info: ['title error', 'description error'], validate_schemas: 'schemas error', validate_unresolved_refs: ['unresolved'], validate_unused_refs: ['unused'], validate_parameter_refs: ['parameter'] } }) - expect(typical.validate()).toEqual(['schemas error', 'unresolved', 'unused', 'parameter']) - const valid = mocked_namespace_file({ groups_errors: [[], []] }) + expect(typical.validate()).toEqual([ + 'schemas error', + 'title error', + 'description error', + 'unresolved', + 'unused', + 'parameter' + ]) + + const valid = mocked_namespace_file({ spec: { info: { title: 'Title', description: 'Description.' } }, groups_errors: [[], []] }) expect(valid.validate()).toEqual([]) }) diff --git a/tools/tests/linter/Operation.test.ts b/tools/tests/linter/Operation.test.ts index aeedd3a00..c5154eb0a 100644 --- a/tools/tests/linter/Operation.test.ts +++ b/tools/tests/linter/Operation.test.ts @@ -69,7 +69,7 @@ test('validate_description()', () => { const invalid_description = operation({ 'x-operation-group': 'indices.create', description: 'This is a description without a period' }) expect(invalid_description.validate_description()) - .toEqual(invalid_description.error('Description must end with a period.')) + .toEqual(invalid_description.error('Description must start with a capital letter and end with a period.')) const valid_description = operation({ 'x-operation-group': 'indices.create', description: 'This is a description with a period.' }) expect(valid_description.validate_description()) diff --git a/tools/tests/linter/Schema.test.ts b/tools/tests/linter/Schema.test.ts index d52492247..afedc8916 100644 --- a/tools/tests/linter/Schema.test.ts +++ b/tools/tests/linter/Schema.test.ts @@ -21,3 +21,11 @@ test('validate_name()', () => { expect(schema('Valid1Name').validate_name()).toBeUndefined() expect(schema('uint').validate_name()).toBeUndefined() }) + +test('validate_description()', () => { + expect(schema('Name', { description: 'Does not end with a period' }).validate_description()).toEqual({ + file: '_common.yaml', + location: '#/components/schemas/Name', + message: "Description must start with a capital letter and end with a period." + }) +}) diff --git a/tools/tests/linter/factories/namespace_file.ts b/tools/tests/linter/factories/namespace_file.ts index 6b661989a..833070c85 100644 --- a/tools/tests/linter/factories/namespace_file.ts +++ b/tools/tests/linter/factories/namespace_file.ts @@ -18,6 +18,7 @@ export function namespace_file (fixture_file: string): NamespaceFile { interface MockedReturnedValues { validate?: string[] validate_name?: string | undefined + validate_info?: string[], validate_schemas?: string | undefined validate_unresolved_refs?: string[] validate_unused_refs?: string[] @@ -41,12 +42,14 @@ export function mocked_namespace_file (ops: { returned_values?: MockedReturnedVa return ns_file } + ns_file.validate_info = jest.fn() ns_file.validate_name = jest.fn() ns_file.validate_schemas = jest.fn() ns_file.validate_unresolved_refs = jest.fn() ns_file.validate_unused_refs = jest.fn() ns_file.validate_parameter_refs = jest.fn() + if (ops.returned_values.validate_info != null) (ns_file.validate_info as jest.Mock).mockReturnValue(ops.returned_values.validate_info) if (ops.returned_values.validate_name != null) (ns_file.validate_name as jest.Mock).mockReturnValue(ops.returned_values.validate_name) if (ops.returned_values.validate_schemas != null) (ns_file.validate_schemas as jest.Mock).mockReturnValue(ops.returned_values.validate_schemas) if (ops.returned_values.validate_unresolved_refs != null) (ns_file.validate_unresolved_refs as jest.Mock).mockReturnValue(ops.returned_values.validate_unresolved_refs) diff --git a/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_capitals.yaml b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_capitals.yaml new file mode 100644 index 000000000..1b58fdd8f --- /dev/null +++ b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_capitals.yaml @@ -0,0 +1,4 @@ +openapi: 3.1.0 +info: + title: Title must be capitalized + description: description must start with a capital letter and end with a period. \ No newline at end of file diff --git a/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_periods.yaml b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_periods.yaml new file mode 100644 index 000000000..877012394 --- /dev/null +++ b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_info_periods.yaml @@ -0,0 +1,4 @@ +openapi: 3.1.0 +info: + title: Title should not have a period. + description: Description should have a period \ No newline at end of file diff --git a/tools/tests/linter/fixtures/file_validators/namespaces/valid_info_multiline_description.yaml b/tools/tests/linter/fixtures/file_validators/namespaces/valid_info_multiline_description.yaml new file mode 100644 index 000000000..eeef80e76 --- /dev/null +++ b/tools/tests/linter/fixtures/file_validators/namespaces/valid_info_multiline_description.yaml @@ -0,0 +1,6 @@ +openapi: 3.1.0 +info: + title: Title is Valid + description: |- + Description is valid. + This description is valid too. \ No newline at end of file diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/cat.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/cat.yaml index 95c4f4c67..46f0228ce 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/cat.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/cat.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/_cat/aliases/{name}': get: diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_a.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_a.yaml index c0feea716..64ece7329 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_a.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_a.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/{index}': {} '/{index}/_clone': {} diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_b.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_b.yaml index 168ecad32..e1f458118 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_b.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_b.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: /_rollover: {} /{index}/_rollover: {} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_c.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_c.yaml index 30d77011d..737089fda 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_c.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/dup_path_c.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/{index}': {} '/{index}/_rollover': {} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/invalid_spec.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/invalid_spec.yaml index 99c22abc8..bf35791a2 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/invalid_spec.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_files/invalid_spec.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/{index}/_doc/{id}': get: diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/cat.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/cat.yaml index 95c4f4c67..46f0228ce 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/cat.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/cat.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/_cat/aliases/{name}': get: diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_a.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_a.yaml index c0feea716..eaf35fd07 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_a.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_a.yaml @@ -1,3 +1,8 @@ + +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/{index}': {} '/{index}/_clone': {} diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_b.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_b.yaml index 168ecad32..e1f458118 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_b.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_b.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: /_rollover: {} /{index}/_rollover: {} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_c.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_c.yaml index 30d77011d..737089fda 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_c.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_c.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/{index}': {} '/{index}/_rollover': {} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_d.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_d.yaml index 200678921..c8be8c315 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_d.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_d.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: '/nodes/{metric}': {} '/nodes/{node_id}': {} diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_a.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_a.yaml index 88c1dc7c6..61babec18 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_a.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_a.yaml @@ -1,6 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Indices API (Namespace 1) + description: Description. version: 1.0.0 paths: '/indices/{metric}': {} diff --git a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_b.yaml b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_b.yaml index 6e8dcd79c..9b363b166 100644 --- a/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_b.yaml +++ b/tools/tests/linter/fixtures/folder_validators/namespaces/invalid_folder/dup_path_namespace_b.yaml @@ -1,6 +1,7 @@ openapi: 3.1.0 info: title: OpenSearch Indices API (Namespace 2) + description: Description. version: 1.0.0 paths: '/indices/{node_id}': {} diff --git a/tools/tests/linter/fixtures/inline_object_schema_validator/namespaces/ops.yaml b/tools/tests/linter/fixtures/inline_object_schema_validator/namespaces/ops.yaml index 0a58a5719..7c8a2a59e 100644 --- a/tools/tests/linter/fixtures/inline_object_schema_validator/namespaces/ops.yaml +++ b/tools/tests/linter/fixtures/inline_object_schema_validator/namespaces/ops.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. paths: /the/path: post: diff --git a/tools/tests/linter/fixtures/schema_refs_validator/namespaces/indices.yaml b/tools/tests/linter/fixtures/schema_refs_validator/namespaces/indices.yaml index f8fc46fb1..4942c1b95 100644 --- a/tools/tests/linter/fixtures/schema_refs_validator/namespaces/indices.yaml +++ b/tools/tests/linter/fixtures/schema_refs_validator/namespaces/indices.yaml @@ -1,3 +1,7 @@ +openapi: 3.1.0 +info: + title: Title + description: Description. components: parameters: animals::query.mammal: diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml index 7b066da2a..8b40f025a 100644 --- a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml @@ -45,7 +45,6 @@ components: type: number2 responses: adopt@200: - description: '' content: application/json: schema: diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml index 4072426cc..85398b8ff 100644 --- a/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml @@ -39,7 +39,6 @@ components: type: number responses: adopt@200: - description: '' application/json: schema: type: object \ No newline at end of file diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml index 81eecb289..d0fa222f7 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml @@ -55,7 +55,6 @@ components: requestBodies: {} responses: info@200: - description: '' content: application/json: schema: @@ -68,7 +67,6 @@ components: required: - tagline info@201: - description: '' content: application/json: schema: @@ -80,7 +78,6 @@ components: - tagline unevaluatedProperties: true info@404: - description: '' content: application/json: schema: @@ -93,7 +90,6 @@ components: unevaluatedProperties: type: object info@500: - description: '' content: application/json: schema: @@ -102,7 +98,6 @@ components: tagline: type: string info@503: - description: '' content: application/json: schema: diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml index 95e119868..cfb4279d2 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml @@ -97,7 +97,6 @@ components: description: Nodes options. responses: info@200: - description: '' content: application/json: schema: @@ -108,7 +107,6 @@ components: required: - tagline info@201: - description: '' content: application/json: schema: @@ -120,7 +118,6 @@ components: - tagline unevaluatedProperties: true info@404: - description: '' content: application/json: schema: @@ -133,7 +130,6 @@ components: unevaluatedProperties: type: object info@500: - description: '' content: application/json: schema: @@ -142,7 +138,6 @@ components: tagline: type: string info@503: - description: '' content: application/json: schema: diff --git a/tools/tests/merger/fixtures/merger/expected.yaml b/tools/tests/merger/fixtures/merger/expected.yaml index ef0560ad2..789d1f203 100644 --- a/tools/tests/merger/fixtures/merger/expected.yaml +++ b/tools/tests/merger/fixtures/merger/expected.yaml @@ -95,12 +95,10 @@ components: type: string responses: adopt@200: - description: '' application/json: schema: type: object indices.create@200: - description: '' application/json: schema: type: object diff --git a/tools/tests/merger/fixtures/spec/namespaces/indices.yaml b/tools/tests/merger/fixtures/spec/namespaces/indices.yaml index d5f1fcb9c..7caa9b02b 100644 --- a/tools/tests/merger/fixtures/spec/namespaces/indices.yaml +++ b/tools/tests/merger/fixtures/spec/namespaces/indices.yaml @@ -41,7 +41,6 @@ components: type: boolean responses: indices.create@200: - description: '' application/json: schema: type: object diff --git a/tools/tests/merger/fixtures/spec/namespaces/shelter.yaml b/tools/tests/merger/fixtures/spec/namespaces/shelter.yaml index 4072426cc..85398b8ff 100644 --- a/tools/tests/merger/fixtures/spec/namespaces/shelter.yaml +++ b/tools/tests/merger/fixtures/spec/namespaces/shelter.yaml @@ -39,7 +39,6 @@ components: type: number responses: adopt@200: - description: '' application/json: schema: type: object \ No newline at end of file diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index a2402b356..68337dc67 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -31,7 +31,6 @@ paths: components: responses: info@200: - description: '' content: application/json: schema: @@ -44,7 +43,6 @@ components: required: - tagline info@201: - description: '' content: application/json: schema: @@ -56,7 +54,6 @@ components: - tagline unevaluatedProperties: true info@404: - description: '' content: application/json: schema: @@ -73,14 +70,13 @@ components: info@removed-2.0: description: Removed in 2.0 via attribute next to ref. info@added-1.3-removed-2.0: - description: 'Added in 1.3, removed in 2.0 via attribute in response body.' + description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' info@500: - description: '' content: application/json: schema: @@ -89,7 +85,6 @@ components: tagline: type: string info@503: - description: '' content: application/json: schema: diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml index 2a2859559..64ad01d39 100644 --- a/tools/tests/tester/fixtures/specs/excerpt.yaml +++ b/tools/tests/tester/fixtures/specs/excerpt.yaml @@ -126,7 +126,6 @@ components: items: type: object cat.indices@200: - description: '' content: text/plain: schema: @@ -152,17 +151,14 @@ components: items: type: object indices.delete@200: - description: '' content: application/json: schema: $ref: '#/components/schemas/_common:IndicesResponseBase' indices.exists@200: - description: '' content: application/json: {} indices.create@200: - description: '' content: application/json: schema: From 59a7ff4f1993bedf0379d58050d154196c63ef85 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 4 Aug 2024 19:29:26 -0400 Subject: [PATCH 03/31] Added support for request headers. (#461) * Renamed request_body to request. Signed-off-by: dblock * Added npm run generate-types. Signed-off-by: dblock * Added support for request headers. Signed-off-by: dblock --------- Signed-off-by: dblock --- CHANGELOG.md | 3 +- TESTING_GUIDE.md | 13 +-- json_schemas/test_story.schema.yaml | 17 +++- package.json | 1 + tests/_core/bulk.yaml | 4 +- tests/_core/mapping.yaml | 2 +- tests/_core/reindex.yaml | 16 ++-- tests/_core/reindex/pipeline.yaml | 8 +- tests/_core/search/_source.yaml | 12 +-- .../search/cancel_after_time_interval.yaml | 2 +- tests/_core/search/geo_distance.yaml | 6 +- tests/_core/search/match.yaml | 6 +- tests/_core/search/multi_match.yaml | 4 +- tests/_core/search/phase_took.yaml | 2 +- tests/_core/search/seq_no_primary_term.yaml | 2 +- tests/_core/search/size.yaml | 6 +- tests/cat/indices.yaml | 2 +- tests/cat/pit_segments/all.yaml | 2 +- tests/cat/pit_segments/pit_segments.yaml | 4 +- tests/cat/repositories.yaml | 2 +- tests/cat/snapshots.yaml | 2 +- tests/cat/templates.yaml | 2 +- tests/cluster/component_template.yaml | 4 +- tests/indices/alias/alias.yaml | 4 +- tests/indices/aliases/aliases.yaml | 2 +- tests/indices/aliases/put_alias.yaml | 6 +- tests/indices/clone.yaml | 2 +- tests/indices/component_template.yaml | 6 +- tests/indices/data_stream/data_stream.yaml | 4 +- tests/indices/data_stream/rollover.yaml | 4 +- tests/indices/data_stream/stats.yaml | 4 +- tests/indices/doc.yaml | 6 +- tests/indices/index.yaml | 2 +- tests/indices/index_template.yaml | 4 +- tests/indices/mapping.yml | 12 +-- tests/indices/refresh.yaml | 2 +- tests/indices/settings.yaml | 2 +- tests/indices/shrink.yaml | 4 +- tests/indices/split.yaml | 12 +-- tests/indices/update_by_query.yaml | 6 +- tests/ingest/pipeline.yaml | 2 +- tests/ml/model_groups.yaml | 4 +- tests/ml/models.yaml | 4 +- tests/ppl/explain.yaml | 4 +- tests/ppl/query.yaml | 4 +- tests/sql/close.yaml | 10 +-- tests/sql/explain.yaml | 6 +- tests/sql/query.yaml | 6 +- tests/sql/settings.yaml | 4 +- tools/src/_utils/SpecificationVisitor.ts | 10 +-- tools/src/linter/SchemasValidator.ts | 4 +- tools/src/linter/components/Operation.ts | 4 +- tools/src/linter/components/OperationGroup.ts | 4 +- tools/src/tester/ChapterEvaluator.ts | 14 +-- tools/src/tester/ChapterReader.ts | 29 +++++-- tools/src/tester/ResultLogger.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 20 ++--- tools/src/tester/_generate_story_types.ts | 2 +- tools/src/tester/types/eval.types.ts | 2 +- tools/src/tester/types/spec.types.ts | 4 +- tools/src/tester/types/story.types.ts | 18 ++-- tools/tests/linter/Operation.test.ts | 6 +- tools/tests/linter/OperationGroup.test.ts | 4 +- tools/tests/linter/factories/operation.ts | 4 +- .../tests/linter/factories/operation_group.ts | 4 +- tools/tests/tester/ChapterReader.test.ts | 85 ++++++++++++++++--- .../fixtures/evals/error/chapter_error.yaml | 2 +- .../fixtures/evals/error/output_error.yaml | 2 +- .../fixtures/evals/failed/invalid_data.yaml | 10 +-- .../fixtures/evals/failed/not_found.yaml | 6 +- tools/tests/tester/fixtures/evals/passed.yaml | 33 +++++-- .../fixtures/fail_non_existent_variable.yaml | 2 +- .../fixtures/stories/failed/invalid_data.yaml | 2 +- .../fixtures/stories/failed/not_found.yaml | 2 +- .../tests/tester/fixtures/stories/passed.yaml | 11 +++ tools/tests/tester/helpers.ts | 2 +- tools/tests/tester/test.test.ts | 6 +- 77 files changed, 335 insertions(+), 209 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bb8bbfd2..361b4f8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,13 +54,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `plugins` to NodeInfoSettings ([#442](https://github.com/opensearch-project/opensearch-api-specification/pull/442)) - Added test coverage ([#443](https://github.com/opensearch-project/opensearch-api-specification/pull/443)) - Added `--opensearch-version` to `merger` that excludes schema elements per semver ([#428](https://github.com/opensearch-project/opensearch-api-specification/pull/428)) -- Added `retry` to `tester` to support asynchronous tasks ([453](https://github.com/opensearch-project/opensearch-api-specification/pull/453)) +- Added `retry` to `tester` to support asynchronous tasks ([#453](https://github.com/opensearch-project/opensearch-api-specification/pull/453)) - Added passing OPENSEARCH_JAVA_OPTS into the docker container used for tests ([#454](https://github.com/opensearch-project/opensearch-api-specification/pull/454)) - Added a warning on mulitple paths being tested in the same file ([#452](https://github.com/opensearch-project/opensearch-api-specification/pull/452)) - Added validation of titles and descriptions in info and schema objects ([#463](https://github.com/opensearch-project/opensearch-api-specification/pull/463)) - Added `/_plugins/_query/settings` ([#456](https://github.com/opensearch-project/opensearch-api-specification/pull/456)) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) - Added tests against OpenSearch 3.0 ([#459](https://github.com/opensearch-project/opensearch-api-specification/pull/459)) +- Added support for request headers in tests [#461](https://github.com/opensearch-project/opensearch-api-specification/pull/461) ### Changed diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 948ce228f..31323efcf 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -75,10 +75,12 @@ chapters: - synopsis: Create an index named `books` with mappings and settings. path: /{index} # The test will fail if "PUT /{index}" operation is not found in the spec. method: PUT - parameters: # All parameters are validated against their schemas in the spec + parameters: # All parameters are validated against their schemas in the spec. index: books - request_body: # The request body is validated against the schema of the requestBody in the spec - payload: + request: # The request. + headers: # Optional headers. + user-agent: OpenSearch API Spec/1.0 + payload: # The request body is validated against the schema of the requestBody in the spec. mappings: properties: name: @@ -88,7 +90,8 @@ chapters: settings: number_of_shards: 5 number_of_replicas: 2 - response: # The response body is validated against the schema of the corresponding response in the spec + response: # The response. + payload: # Matching response payload. The entire payload is validated against the schema of the corresponding response in the spec. status: 200 # This is the expected status code of the response. Any other status code will fail the test. - synopsis: Retrieve the mappings and settings of the `books` index. @@ -113,7 +116,7 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) id: create_model_group # Only needed if you want to refer to this chapter in another chapter. path: /_plugins/_ml/model_groups/_register method: POST - request_body: + request: payload: name: NLP_Group description: Model group for NLP models. diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index d0b09e5ce..829eb5286 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -77,8 +77,8 @@ definitions: type: object additionalProperties: $ref: '#/definitions/Parameter' - request_body: - $ref: '#/definitions/RequestBody' + request: + $ref: '#/definitions/Request' output: $ref: '#/definitions/Output' version: @@ -119,15 +119,18 @@ definitions: required: - count - RequestBody: + Request: type: object properties: content_type: type: string default: application/json + headers: + type: object + additionalProperties: + $ref: '#/definitions/Header' payload: $ref: '#/definitions/Payload' - required: [payload] additionalProperties: false ExpectedResponse: @@ -163,6 +166,12 @@ definitions: required: [content_type, payload, status] additionalProperties: false + Header: + anyOf: + - type: string + - type: number + - type: boolean + Payload: anyOf: - type: object diff --git a/package.json b/package.json index 04e7116e4..d73f0868d 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "scripts": { "coverage:spec": "ts-node tools/src/coverage/coverage.ts", "dump-cluster-spec": "ts-node tools/src/dump-cluster-spec/dump-cluster-spec.ts", + "generate-types": "ts-node tools/src/tester/_generate_story_types.ts", "lint:spec": "ts-node tools/src/linter/lint.ts", "lint": "eslint . --report-unused-disable-directives", "lint--fix": "eslint . --fix --report-unused-disable-directives", diff --git a/tests/_core/bulk.yaml b/tests/_core/bulk.yaml index c480b926e..abcbb438f 100644 --- a/tests/_core/bulk.yaml +++ b/tests/_core/bulk.yaml @@ -9,7 +9,7 @@ chapters: - synopsis: Create an index. path: /_bulk method: POST - request_body: + request: content_type: application/x-ndjson payload: - {create: {_index: movies}} @@ -17,7 +17,7 @@ chapters: - synopsis: Bulk document CRUD. path: /_bulk method: POST - request_body: + request: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} diff --git a/tests/_core/mapping.yaml b/tests/_core/mapping.yaml index 99973978f..936cf021e 100644 --- a/tests/_core/mapping.yaml +++ b/tests/_core/mapping.yaml @@ -6,7 +6,7 @@ prologues: method: PUT parameters: index: movies - request_body: + request: payload: mappings: properties: diff --git a/tests/_core/reindex.yaml b/tests/_core/reindex.yaml index 1cf82fde1..2b349d341 100644 --- a/tests/_core/reindex.yaml +++ b/tests/_core/reindex.yaml @@ -17,7 +17,7 @@ prologues: parameters: index: movies refresh: true - request_body: + request: payload: title: Beauty and the Beast year: 91 @@ -26,7 +26,7 @@ chapters: - synopsis: Reindex from movies to films. path: /_reindex method: POST - request_body: + request: payload: source: index: movies @@ -39,7 +39,7 @@ chapters: - synopsis: Reindex a subset of documents (match). path: /_reindex method: POST - request_body: + request: payload: source: index: movies @@ -55,7 +55,7 @@ chapters: - synopsis: Reindex a subset of documents (no match). path: /_reindex method: POST - request_body: + request: payload: source: index: movies @@ -71,7 +71,7 @@ chapters: - synopsis: Combine two indexes. path: /_reindex method: POST - request_body: + request: payload: source: index: @@ -86,7 +86,7 @@ chapters: - synopsis: Reindex only unique documents. path: /_reindex method: POST - request_body: + request: payload: conflicts: proceed source: @@ -101,7 +101,7 @@ chapters: - synopsis: Transform documents during reindex. path: /_reindex method: POST - request_body: + request: payload: source: index: movies @@ -120,7 +120,7 @@ chapters: parameters: max_docs: 1 slices: 1 - request_body: + request: payload: source: index: movies diff --git a/tests/_core/reindex/pipeline.yaml b/tests/_core/reindex/pipeline.yaml index f8e263ae7..fbb6e1065 100644 --- a/tests/_core/reindex/pipeline.yaml +++ b/tests/_core/reindex/pipeline.yaml @@ -14,7 +14,7 @@ epilogues: prologues: - path: /_ingest/pipeline/transform-and-count method: PUT - request_body: + request: payload: description: | Splits the `title`` field into a `words` list. @@ -35,7 +35,7 @@ prologues: parameters: index: movies refresh: true - request_body: + request: payload: title: Beauty and the Beast year: 91 @@ -44,7 +44,7 @@ chapters: - synopsis: Transform documents using a pipeline. path: /_reindex method: POST - request_body: + request: payload: source: index: movies @@ -67,7 +67,7 @@ chapters: method: POST parameters: index: videos - request_body: + request: payload: query: match_all: {} diff --git a/tests/_core/search/_source.yaml b/tests/_core/search/_source.yaml index 973eb3585..de34a143b 100644 --- a/tests/_core/search/_source.yaml +++ b/tests/_core/search/_source.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball @@ -22,7 +22,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: _source: true query: @@ -48,7 +48,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: _source: false query: @@ -70,7 +70,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: _source: - director @@ -97,7 +97,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: _source: includes: director @@ -123,7 +123,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: _source: includes: '*' diff --git a/tests/_core/search/cancel_after_time_interval.yaml b/tests/_core/search/cancel_after_time_interval.yaml index 012a7972e..b5bd3b510 100644 --- a/tests/_core/search/cancel_after_time_interval.yaml +++ b/tests/_core/search/cancel_after_time_interval.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball diff --git a/tests/_core/search/geo_distance.yaml b/tests/_core/search/geo_distance.yaml index 8ce5712d0..844df33c2 100644 --- a/tests/_core/search/geo_distance.yaml +++ b/tests/_core/search/geo_distance.yaml @@ -4,7 +4,7 @@ description: Test search endpoint with geo_distance query. prologues: - path: /map method: PUT - request_body: + request: payload: mappings: properties: @@ -14,7 +14,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: field: lat: 74 @@ -30,7 +30,7 @@ chapters: parameters: index: map method: GET - request_body: + request: payload: query: geo_distance: diff --git a/tests/_core/search/match.yaml b/tests/_core/search/match.yaml index 78afe3ddf..2766f3ac1 100644 --- a/tests/_core/search/match.yaml +++ b/tests/_core/search/match.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball @@ -22,7 +22,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: size: 1 query: @@ -34,7 +34,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: size: 1 query: diff --git a/tests/_core/search/multi_match.yaml b/tests/_core/search/multi_match.yaml index 4ea41f74a..b5c7b3c76 100644 --- a/tests/_core/search/multi_match.yaml +++ b/tests/_core/search/multi_match.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball @@ -22,7 +22,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: size: 1 query: diff --git a/tests/_core/search/phase_took.yaml b/tests/_core/search/phase_took.yaml index 846d0fe86..3e061ae0f 100644 --- a/tests/_core/search/phase_took.yaml +++ b/tests/_core/search/phase_took.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball diff --git a/tests/_core/search/seq_no_primary_term.yaml b/tests/_core/search/seq_no_primary_term.yaml index 07b0d5834..8082de549 100644 --- a/tests/_core/search/seq_no_primary_term.yaml +++ b/tests/_core/search/seq_no_primary_term.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball diff --git a/tests/_core/search/size.yaml b/tests/_core/search/size.yaml index 154f34638..64b2b8ca9 100644 --- a/tests/_core/search/size.yaml +++ b/tests/_core/search/size.yaml @@ -6,7 +6,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: director: Bennett Miller title: Moneyball @@ -22,7 +22,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: size: 0 response: @@ -40,7 +40,7 @@ chapters: parameters: index: movies method: POST - request_body: + request: payload: size: 1 query: diff --git a/tests/cat/indices.yaml b/tests/cat/indices.yaml index 90c7d81b5..c8d3fcb42 100644 --- a/tests/cat/indices.yaml +++ b/tests/cat/indices.yaml @@ -6,7 +6,7 @@ prologues: method: PUT parameters: index: books - request_body: + request: payload: {} epilogues: - path: /books diff --git a/tests/cat/pit_segments/all.yaml b/tests/cat/pit_segments/all.yaml index 5b8a1f9ff..91b4ccdd1 100644 --- a/tests/cat/pit_segments/all.yaml +++ b/tests/cat/pit_segments/all.yaml @@ -11,7 +11,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: title: Monopoly status: [201] diff --git a/tests/cat/pit_segments/pit_segments.yaml b/tests/cat/pit_segments/pit_segments.yaml index 172988830..21ea56af0 100644 --- a/tests/cat/pit_segments/pit_segments.yaml +++ b/tests/cat/pit_segments/pit_segments.yaml @@ -11,7 +11,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: payload: title: Monopoly status: [201] @@ -30,7 +30,7 @@ chapters: method: GET parameters: format: json - request_body: + request: payload: pit_id: - ${create_pit.pit_id} diff --git a/tests/cat/repositories.yaml b/tests/cat/repositories.yaml index 80222e5ba..72cd80dce 100644 --- a/tests/cat/repositories.yaml +++ b/tests/cat/repositories.yaml @@ -12,7 +12,7 @@ prologues: method: PUT parameters: repository: my-fs-repository - request_body: + request: payload: type: fs settings: diff --git a/tests/cat/snapshots.yaml b/tests/cat/snapshots.yaml index 1163c5db5..962827112 100644 --- a/tests/cat/snapshots.yaml +++ b/tests/cat/snapshots.yaml @@ -8,7 +8,7 @@ epilogues: prologues: - path: /_snapshot/my-fs-repository method: PUT - request_body: + request: payload: type: fs settings: diff --git a/tests/cat/templates.yaml b/tests/cat/templates.yaml index b72262fe7..3f233a76b 100644 --- a/tests/cat/templates.yaml +++ b/tests/cat/templates.yaml @@ -8,7 +8,7 @@ epilogues: prologues: - path: /_index_template/daily_logs method: PUT - request_body: + request: payload: index_patterns: - 'logs*' diff --git a/tests/cluster/component_template.yaml b/tests/cluster/component_template.yaml index 05838d5db..783393591 100644 --- a/tests/cluster/component_template.yaml +++ b/tests/cluster/component_template.yaml @@ -11,7 +11,7 @@ chapters: method: POST parameters: name: template1 - request_body: + request: payload: template: settings: @@ -31,7 +31,7 @@ chapters: method: PUT parameters: name: template1 - request_body: + request: payload: template: settings: diff --git a/tests/indices/alias/alias.yaml b/tests/indices/alias/alias.yaml index 5bd41501c..09962aff6 100644 --- a/tests/indices/alias/alias.yaml +++ b/tests/indices/alias/alias.yaml @@ -48,7 +48,7 @@ chapters: parameters: index: games name: jeux - request_body: + request: payload: index_routing: test1 search_routing: test2 @@ -63,6 +63,6 @@ chapters: parameters: index: games name: jeux - request_body: + request: payload: is_hidden: true diff --git a/tests/indices/aliases/aliases.yaml b/tests/indices/aliases/aliases.yaml index a1bc2baac..22bcd95f1 100644 --- a/tests/indices/aliases/aliases.yaml +++ b/tests/indices/aliases/aliases.yaml @@ -18,7 +18,7 @@ chapters: - synopsis: Multiple alias operations. path: /_aliases method: POST - request_body: + request: payload: actions: - remove: diff --git a/tests/indices/aliases/put_alias.yaml b/tests/indices/aliases/put_alias.yaml index 7dd73fe9c..ea65d3c53 100644 --- a/tests/indices/aliases/put_alias.yaml +++ b/tests/indices/aliases/put_alias.yaml @@ -30,7 +30,7 @@ chapters: method: PUT parameters: index: test_index - request_body: + request: payload: alias: test_alias_1 response: @@ -43,7 +43,7 @@ chapters: method: PUT parameters: name: test_alias_2 - request_body: + request: payload: index: test_index response: @@ -54,7 +54,7 @@ chapters: - synopsis: Create an alias - specifying index & alias in body. path: /_alias method: PUT - request_body: + request: payload: index: test_index alias: test_alias_3 diff --git a/tests/indices/clone.yaml b/tests/indices/clone.yaml index f3132ec78..dfc56119c 100644 --- a/tests/indices/clone.yaml +++ b/tests/indices/clone.yaml @@ -6,7 +6,7 @@ prologues: method: PUT - path: /movies/_settings method: PUT - request_body: + request: payload: settings: index.blocks.write: true diff --git a/tests/indices/component_template.yaml b/tests/indices/component_template.yaml index c77477f0c..d05b88f4c 100644 --- a/tests/indices/component_template.yaml +++ b/tests/indices/component_template.yaml @@ -20,7 +20,7 @@ chapters: method: POST parameters: name: component_template_1 - request_body: + request: payload: template: mappings: @@ -41,7 +41,7 @@ chapters: method: POST parameters: name: component_template_2 - request_body: + request: payload: template: mappings: @@ -57,7 +57,7 @@ chapters: method: POST parameters: name: daily_logs - request_body: + request: payload: index_patterns: - 'logs*' diff --git a/tests/indices/data_stream/data_stream.yaml b/tests/indices/data_stream/data_stream.yaml index d4b7952b2..051451286 100644 --- a/tests/indices/data_stream/data_stream.yaml +++ b/tests/indices/data_stream/data_stream.yaml @@ -4,7 +4,7 @@ description: Test data streams. prologues: - path: /_index_template/logs-template-nginx method: PUT - request_body: + request: payload: index_patterns: - logs-* @@ -33,7 +33,7 @@ chapters: method: POST parameters: index: logs-nginx - request_body: + request: payload: message: login attempt failed request_time: '2013-03-01T00:00:00' diff --git a/tests/indices/data_stream/rollover.yaml b/tests/indices/data_stream/rollover.yaml index 2607b1c69..f8640db10 100644 --- a/tests/indices/data_stream/rollover.yaml +++ b/tests/indices/data_stream/rollover.yaml @@ -4,7 +4,7 @@ description: Test data streams rollover. prologues: - path: /_index_template/logs-template-nginx method: PUT - request_body: + request: payload: index_patterns: - logs-* @@ -17,7 +17,7 @@ prologues: method: PUT - path: /logs-nginx/_doc method: POST - request_body: + request: payload: message: login attempt failed request_time: '2013-03-01T00:00:00' diff --git a/tests/indices/data_stream/stats.yaml b/tests/indices/data_stream/stats.yaml index c963997be..8052ff670 100644 --- a/tests/indices/data_stream/stats.yaml +++ b/tests/indices/data_stream/stats.yaml @@ -4,7 +4,7 @@ description: Test data streams. prologues: - path: /_index_template/logs-template-nginx method: PUT - request_body: + request: payload: index_patterns: - logs-* @@ -17,7 +17,7 @@ prologues: method: PUT - path: /logs-nginx/_doc method: POST - request_body: + request: payload: message: login attempt failed request_time: '2013-03-01T00:00:00' diff --git a/tests/indices/doc.yaml b/tests/indices/doc.yaml index cf01403b8..325602d75 100644 --- a/tests/indices/doc.yaml +++ b/tests/indices/doc.yaml @@ -11,7 +11,7 @@ chapters: method: POST parameters: index: movies - request_body: + request: payload: title: Beauty and the Beast year: 1991 @@ -23,7 +23,7 @@ chapters: parameters: index: movies id: '1' - request_body: + request: payload: title: Beauty and the Beast (Id) year: 1991 @@ -35,7 +35,7 @@ chapters: parameters: index: movies id: '1' - request_body: + request: payload: title: Beauty and the Beast (Updated) year: 1991 diff --git a/tests/indices/index.yaml b/tests/indices/index.yaml index 22ee0a3d7..186b7df8f 100644 --- a/tests/indices/index.yaml +++ b/tests/indices/index.yaml @@ -14,7 +14,7 @@ chapters: method: PUT parameters: index: books - request_body: + request: payload: mappings: properties: diff --git a/tests/indices/index_template.yaml b/tests/indices/index_template.yaml index c61011894..09a6ce7f4 100644 --- a/tests/indices/index_template.yaml +++ b/tests/indices/index_template.yaml @@ -11,7 +11,7 @@ chapters: method: POST parameters: name: daily_logs - request_body: + request: payload: index_patterns: - 'logs*' @@ -27,7 +27,7 @@ chapters: method: PUT parameters: name: daily_logs - request_body: + request: payload: index_patterns: - 'logs*' diff --git a/tests/indices/mapping.yml b/tests/indices/mapping.yml index af2b29518..00a11407d 100644 --- a/tests/indices/mapping.yml +++ b/tests/indices/mapping.yml @@ -10,7 +10,7 @@ prologues: method: PUT parameters: index: movies - request_body: + request: payload: mappings: properties: @@ -68,7 +68,7 @@ chapters: ignore_unavailable: true timeout: 10s write_index_only: true - request_body: + request: payload: dynamic: 'true' properties: @@ -85,7 +85,7 @@ chapters: parameters: index: movies cluster_manager_timeout: 1s - request_body: + request: payload: properties: producer: @@ -104,7 +104,7 @@ chapters: ignore_unavailable: true timeout: 10s write_index_only: true - request_body: + request: payload: dynamic: 'false' response: @@ -121,7 +121,7 @@ chapters: ignore_unavailable: true timeout: 10s write_index_only: true - request_body: + request: payload: dynamic: strict response: @@ -139,7 +139,7 @@ chapters: ignore_unavailable: true timeout: 10s write_index_only: true - request_body: + request: payload: dynamic: strict_allow_templates response: diff --git a/tests/indices/refresh.yaml b/tests/indices/refresh.yaml index d4aff0cef..587168962 100644 --- a/tests/indices/refresh.yaml +++ b/tests/indices/refresh.yaml @@ -8,7 +8,7 @@ epilogues: prologues: - path: /movies/_doc method: POST - request_body: + request: payload: title: Beauty and the Beast year: 1991 diff --git a/tests/indices/settings.yaml b/tests/indices/settings.yaml index dc5a85f7b..a5835fdb0 100644 --- a/tests/indices/settings.yaml +++ b/tests/indices/settings.yaml @@ -41,7 +41,7 @@ chapters: method: PUT parameters: index: movies - request_body: + request: payload: settings: index.blocks.write: true diff --git a/tests/indices/shrink.yaml b/tests/indices/shrink.yaml index 58b8fee23..1803889ab 100644 --- a/tests/indices/shrink.yaml +++ b/tests/indices/shrink.yaml @@ -4,13 +4,13 @@ description: Test shrinking an index. prologues: - path: /movies method: PUT - request_body: + request: payload: settings: index.number_of_shards: 3 - path: /movies/_settings method: PUT - request_body: + request: payload: settings: index.blocks.write: true diff --git a/tests/indices/split.yaml b/tests/indices/split.yaml index ae6204060..980feb287 100644 --- a/tests/indices/split.yaml +++ b/tests/indices/split.yaml @@ -4,13 +4,13 @@ description: Test splitting an index. prologues: - path: /movies method: PUT - request_body: + request: payload: settings: index.number_of_shards: 3 - path: /movies/_settings method: PUT - request_body: + request: payload: settings: index.blocks.write: true @@ -39,7 +39,7 @@ chapters: target: movies1 wait_for_active_shards: 1 timeout: 10s - request_body: + request: payload: settings: index: @@ -58,7 +58,7 @@ chapters: index: movies target: movies2 wait_for_completion: true - request_body: + request: payload: settings: index: @@ -73,7 +73,7 @@ chapters: target: movies3 wait_for_active_shards: 1 timeout: 10s - request_body: + request: payload: settings: index: @@ -94,7 +94,7 @@ chapters: wait_for_active_shards: 1 cluster_manager_timeout: 10s timeout: 10s - request_body: + request: payload: settings: index: diff --git a/tests/indices/update_by_query.yaml b/tests/indices/update_by_query.yaml index 4043dc496..a58853f3f 100644 --- a/tests/indices/update_by_query.yaml +++ b/tests/indices/update_by_query.yaml @@ -10,7 +10,7 @@ prologues: method: POST parameters: refresh: true - request_body: + request: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} @@ -33,7 +33,7 @@ chapters: method: POST parameters: index: books - request_body: + request: payload: query: term: @@ -56,7 +56,7 @@ chapters: method: POST parameters: index: books - request_body: + request: payload: query: term: diff --git a/tests/ingest/pipeline.yaml b/tests/ingest/pipeline.yaml index ee8e539ea..3319d7026 100644 --- a/tests/ingest/pipeline.yaml +++ b/tests/ingest/pipeline.yaml @@ -13,7 +13,7 @@ chapters: method: PUT parameters: id: books_pipeline - request_body: + request: payload: description: Extracts text from field and embeds it. processors: diff --git a/tests/ml/model_groups.yaml b/tests/ml/model_groups.yaml index 46f8595d7..778863820 100644 --- a/tests/ml/model_groups.yaml +++ b/tests/ml/model_groups.yaml @@ -6,7 +6,7 @@ version: '>= 2.11' prologues: - path: /_cluster/settings method: PUT - request_body: + request: payload: persistent: plugins: @@ -25,7 +25,7 @@ chapters: id: create_model_group path: /_plugins/_ml/model_groups/_register method: POST - request_body: + request: payload: name: NLP_Group description: Model group for NLP models diff --git a/tests/ml/models.yaml b/tests/ml/models.yaml index 452f7bacf..f82d2ecda 100644 --- a/tests/ml/models.yaml +++ b/tests/ml/models.yaml @@ -6,7 +6,7 @@ version: '>= 2.11' prologues: - path: /_cluster/settings method: PUT - request_body: + request: payload: persistent: plugins: @@ -17,7 +17,7 @@ chapters: id: create_model path: /_plugins/_ml/models/_register method: POST - request_body: + request: payload: name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b version: 1.0.1 diff --git a/tests/ppl/explain.yaml b/tests/ppl/explain.yaml index 0dbc465ea..89421fb70 100644 --- a/tests/ppl/explain.yaml +++ b/tests/ppl/explain.yaml @@ -5,7 +5,7 @@ description: Test how a query is executed against OpenSearch. prologues: - path: /books method: PUT - request_body: + request: payload: {} epilogues: - path: /books @@ -15,7 +15,7 @@ chapters: - synopsis: Get explain of SQL Query path: /_plugins/_ppl/_explain method: POST - request_body: + request: payload: query: search source=books response: diff --git a/tests/ppl/query.yaml b/tests/ppl/query.yaml index 53a9720d3..b9b3548be 100644 --- a/tests/ppl/query.yaml +++ b/tests/ppl/query.yaml @@ -7,7 +7,7 @@ prologues: method: PUT parameters: index: books - request_body: + request: payload: {} epilogues: - path: /books @@ -17,7 +17,7 @@ chapters: - synopsis: Get PPL query path: /_plugins/_ppl method: POST - request_body: + request: payload: query: search source=books response: diff --git a/tests/sql/close.yaml b/tests/sql/close.yaml index df2910ae9..0f70d62c8 100644 --- a/tests/sql/close.yaml +++ b/tests/sql/close.yaml @@ -7,11 +7,11 @@ prologues: method: PUT parameters: index: books - request_body: + request: payload: {} - path: /_bulk method: POST - request_body: + request: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} @@ -24,14 +24,14 @@ prologues: index: books - path: _plugins/_query/settings method: PUT - request_body: + request: payload: transient: plugins.query.memory_limit: 100% - id: sql_query path: /_plugins/_sql method: POST - request_body: + request: payload: fetch_size: 1 query: 'SELECT * FROM books' @@ -45,7 +45,7 @@ chapters: - synopsis: Close cursor. path: /_plugins/_sql/close method: POST - request_body: + request: payload: cursor: ${sql_query.cursor} response: diff --git a/tests/sql/explain.yaml b/tests/sql/explain.yaml index 3679dcb29..245d65d83 100644 --- a/tests/sql/explain.yaml +++ b/tests/sql/explain.yaml @@ -5,11 +5,11 @@ description: Test how a query is executed against OpenSearch. prologues: - path: /books method: PUT - request_body: + request: payload: {} - path: _plugins/_query/settings method: PUT - request_body: + request: payload: transient: plugins.query.memory_limit: 100% @@ -21,7 +21,7 @@ chapters: - synopsis: Get explain of SQL Query path: /_plugins/_sql/_explain method: POST - request_body: + request: payload: query: SELECT * FROM books response: diff --git a/tests/sql/query.yaml b/tests/sql/query.yaml index e66942606..ba1d34317 100644 --- a/tests/sql/query.yaml +++ b/tests/sql/query.yaml @@ -7,11 +7,11 @@ prologues: method: PUT parameters: index: books - request_body: + request: payload: {} - path: _plugins/_query/settings method: PUT - request_body: + request: payload: transient: plugins.query.memory_limit: 100% @@ -25,7 +25,7 @@ chapters: method: POST parameters: sanitize: false - request_body: + request: payload: query: 'SELECT * FROM books' response: diff --git a/tests/sql/settings.yaml b/tests/sql/settings.yaml index 68e5f91c7..6a826806f 100644 --- a/tests/sql/settings.yaml +++ b/tests/sql/settings.yaml @@ -6,7 +6,7 @@ chapters: - synopsis: Update SQL settings. path: /_plugins/_query/settings method: PUT - request_body: + request: payload: transient: plugins: @@ -25,7 +25,7 @@ chapters: - synopsis: Update SQL settings with plain request body. path: /_plugins/_query/settings method: PUT - request_body: + request: payload: transient: plugins.sql.enabled: true diff --git a/tools/src/_utils/SpecificationVisitor.ts b/tools/src/_utils/SpecificationVisitor.ts index b97a636e4..ee8fb276d 100644 --- a/tools/src/_utils/SpecificationVisitor.ts +++ b/tools/src/_utils/SpecificationVisitor.ts @@ -58,13 +58,13 @@ export class SpecificationVisitor { visit_operation (ctx: SpecificationContext, operation: OpenAPIV3.OperationObject): void { visit_each(ctx, operation, 'parameters', this.visit_parameter.bind(this)) - visit(ctx, operation, 'requestBody', this.visit_request_body.bind(this)) + visit(ctx, operation, 'requestBody', this.visit_request.bind(this)) visit_each(ctx, operation, 'responses', this.visit_response.bind(this)) } visit_components (ctx: SpecificationContext, components: OpenAPIV3.ComponentsObject): void { visit_each(ctx, components, 'parameters', this.visit_parameter.bind(this)) - visit_each(ctx, components, 'requestBodies', this.visit_request_body.bind(this)) + visit_each(ctx, components, 'requestBodies', this.visit_request.bind(this)) visit_each(ctx, components, 'responses', this.visit_response.bind(this)) visit_each(ctx, components, 'schemas', this.visit_schema.bind(this)) } @@ -75,10 +75,10 @@ export class SpecificationVisitor { visit(ctx, parameter, 'schema', this.visit_schema.bind(this)) } - visit_request_body (ctx: SpecificationContext, request_body: MaybeRef): void { - if (is_ref(request_body)) return + visit_request (ctx: SpecificationContext, request: MaybeRef): void { + if (is_ref(request)) return - visit_each(ctx, request_body, 'content', this.visit_media_type.bind(this)) + visit_each(ctx, request, 'content', this.visit_media_type.bind(this)) } visit_response (ctx: SpecificationContext, response: MaybeRef): void { diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index d22c0beaa..06672285b 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -38,7 +38,7 @@ export default class SchemasValidator { if (named_schemas_errors.length > 0) return named_schemas_errors return [ ...this.validate_parameter_schemas(), - ...this.validate_request_body_schemas(), + ...this.validate_request_schemas(), ...this.validate_response_schemas() ] } @@ -66,7 +66,7 @@ export default class SchemasValidator { }).filter((error) => error != null) as ValidationError[] } - validate_request_body_schemas (): ValidationError[] { + validate_request_schemas (): ValidationError[] { return Object.entries(this.spec.requestBodies as Record).flatMap(([namespace, body]) => { const file = `namespaces/${namespace}.yaml` const location = `#/components/requestBodies/${namespace}` diff --git a/tools/src/linter/components/Operation.ts b/tools/src/linter/components/Operation.ts index bb06233ad..f55b29e27 100644 --- a/tools/src/linter/components/Operation.ts +++ b/tools/src/linter/components/Operation.ts @@ -40,7 +40,7 @@ export default class Operation extends ValidatorBase { return [ this.validate_operation_id(), this.validate_description(), - this.validate_request_body(), + this.validate_request(), this.validate_parameters(), this.validate_path_parameters(), this.validate_order_of_parameters(), @@ -77,7 +77,7 @@ export default class Operation extends ValidatorBase { if (!regex.test(id)) { return this.error(`Invalid operationId '${id}'. Must be in {x-operation-group}.{number} format.`) } } - validate_request_body (): ValidationError | undefined { + validate_request (): ValidationError | undefined { const body = this.spec.requestBody if (!body) return const expected = `#/components/requestBodies/${this.group}` diff --git a/tools/src/linter/components/OperationGroup.ts b/tools/src/linter/components/OperationGroup.ts index 5daa6adce..dc45765bc 100644 --- a/tools/src/linter/components/OperationGroup.ts +++ b/tools/src/linter/components/OperationGroup.ts @@ -32,7 +32,7 @@ export default class OperationGroup extends ValidatorBase { return [ this.validate_description(), this.validate_external_docs(), - this.validate_request_body(), + this.validate_request(), this.validate_responses(), this.validate_query_parameters() ].filter((e) => e) as ValidationError[] @@ -48,7 +48,7 @@ export default class OperationGroup extends ValidatorBase { if (uniq_external_docs.size > 1) { return this.error(`${this.operations.length} '${this.name}' operations must have identical externalDocs property.`) } } - validate_request_body (): ValidationError | undefined { + validate_request (): ValidationError | undefined { const uniq_request_bodies = new Set(this.operations.map((op) => op.spec.requestBody?.$ref)) if (uniq_request_bodies.size > 1) { return this.error(`${this.operations.length} '${this.name}' operations must have identical requestBody property.`) } } diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 43157d2b8..e06fca445 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -64,7 +64,7 @@ export default class ChapterEvaluator { async #evaluate(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise { const response = await this._chapter_reader.read(chapter, story_outputs) const params = this.#evaluate_parameters(chapter, operation) - const request_body = this.#evaluate_request_body(chapter, operation) + const request = this.#evaluate_request(chapter, operation) const status = this.#evaluate_status(chapter, response) const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED } const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } @@ -72,7 +72,7 @@ export default class ChapterEvaluator { const evaluations = _.compact(_.concat( Object.values(params), - request_body, + request, status, payload_body_evaluation, payload_schema_evaluation, @@ -83,7 +83,7 @@ export default class ChapterEvaluator { title: chapter.synopsis, path: `${chapter.method} ${chapter.path}`, overall: { result: overall_result(evaluations) }, - request: { parameters: params, request_body }, + request: { parameters: params, request }, retries, response: { status, @@ -109,12 +109,12 @@ export default class ChapterEvaluator { })) } - #evaluate_request_body(chapter: Chapter, operation: ParsedOperation): Evaluation { - if (!chapter.request_body) return { result: Result.PASSED } - const content_type = chapter.request_body.content_type ?? APPLICATION_JSON + #evaluate_request(chapter: Chapter, operation: ParsedOperation): Evaluation { + if (chapter.request?.payload === undefined) return { result: Result.PASSED } + const content_type = chapter.request.content_type ?? APPLICATION_JSON const schema = operation.requestBody?.content[content_type]?.schema if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` } - return this._schema_validator.validate(schema, chapter.request_body?.payload ?? {}) + return this._schema_validator.validate(schema, chapter.request?.payload ?? {}) } #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation { diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 9467976d0..ae26e9920 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -17,6 +17,7 @@ import YAML from 'yaml' import CBOR from 'cbor' import SMILE from 'smile-js' import { APPLICATION_CBOR, APPLICATION_JSON, APPLICATION_SMILE, APPLICATION_YAML, TEXT_PLAIN } from "./MimeTypes"; +import _ from 'lodash' export default class ChapterReader { private readonly _client: OpenSearchHttpClient @@ -31,16 +32,16 @@ export default class ChapterReader { const response: Record = {} const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {}) const [url_path, params] = this.#parse_url(chapter.path, resolved_params) - const content_type = chapter.request_body?.content_type ?? APPLICATION_JSON - const request_data = chapter.request_body?.payload !== undefined ? this.#serialize_payload( - story_outputs.resolve_value(chapter.request_body.payload), + const [headers, content_type] = this.#serialize_headers(chapter.request?.headers, chapter.request?.content_type) + const request_data = chapter.request?.payload !== undefined ? this.#serialize_payload( + story_outputs.resolve_value(chapter.request.payload), content_type ) : undefined - this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] | ${to_json(request_data)}`) + this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] ${_.compact([to_json(headers), to_json(request_data)]).join(' | ')}`) await this._client.request({ url: url_path, method: chapter.method, - headers: { 'Content-Type' : content_type }, + headers: { 'Content-Type' : content_type, ...headers }, params, data: request_data, paramsSerializer: (params) => { // eslint-disable-line @typescript-eslint/naming-convention @@ -49,7 +50,8 @@ export default class ChapterReader { }).then(r => { response.status = r.status response.content_type = r.headers['content-type']?.split(';')[0] - response.payload = this.#deserialize_payload(r.data, response.content_type) + const payload = this.#deserialize_payload(r.data, response.content_type) + if (payload !== undefined) response.payload = payload this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(response.payload)}`) }).catch(e => { if (e.response == null) { @@ -59,7 +61,7 @@ export default class ChapterReader { response.status = e.response.status response.content_type = e.response.headers['content-type']?.split(';')[0] const payload = this.#deserialize_payload(e.response.data, response.content_type) - response.payload = payload?.error + if (payload !== undefined) response.payload = payload.error response.message = payload.error?.reason ?? e.response.statusText response.error = e @@ -68,6 +70,19 @@ export default class ChapterReader { return response as ActualResponse } + #serialize_headers(headers?: Record, content_type?: string): [Record | undefined, string] { + headers = _.cloneDeep(headers) + content_type = content_type ?? APPLICATION_JSON + if (!headers) return [headers, content_type] + _.forEach(headers, (v, k) => { + if (k.toLowerCase() == 'content-type') { + content_type = v.toString() + if (headers) delete headers[k] + } + }) + return [headers, content_type] + } + #serialize_payload(payload: any, content_type: string): any { if (payload === undefined) return undefined switch (content_type) { diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts index ddfead55c..d2ef339c2 100644 --- a/tools/src/tester/ResultLogger.ts +++ b/tools/src/tester/ResultLogger.ts @@ -62,7 +62,7 @@ export class ConsoleResultLogger implements ResultLogger { #log_chapter (chapter: ChapterEvaluation): void { this.#log_evaluation(chapter.overall, ansi.i(chapter.title), this._tab_width * 2) this.#log_parameters(chapter.request?.parameters ?? {}) - this.#log_request_body(chapter.request?.request_body) + this.#log_request(chapter.request?.request) this.#log_status(chapter.response?.status) this.#log_payload_body(chapter.response?.payload_body) this.#log_payload_schema(chapter.response?.payload_schema) @@ -79,7 +79,7 @@ export class ConsoleResultLogger implements ResultLogger { } } - #log_request_body (evaluation: Evaluation | undefined): void { + #log_request (evaluation: Evaluation | undefined): void { if (evaluation == null) return this.#log_evaluation(evaluation, 'REQUEST BODY', this._tab_width * 3) } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 1e2a95d52..fede2e397 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -170,7 +170,7 @@ export default class StoryEvaluator { const variables = new Set() const title = `${chapter.method} ${chapter.path}` StoryEvaluator.#extract_params_variables(chapter.parameters ?? {}, variables) - StoryEvaluator.#extract_request_body_variables(chapter.request_body?.payload ?? {}, variables) + StoryEvaluator.#extract_request_variables(chapter.request?.payload ?? {}, variables) for (const { chapter_id, output_name } of variables) { if (!story_outputs.has_chapter(chapter_id)) { return StoryEvaluator.#failed_evaluation(title, `Chapter makes reference to non existent chapter "${chapter_id}`) @@ -192,24 +192,24 @@ export default class StoryEvaluator { }) } - static #extract_request_body_variables(request_body: any, variables: Set): void { - const request_body_type = typeof request_body - switch (request_body_type) { + static #extract_request_variables(request: any, variables: Set): void { + const request_type = typeof request + switch (request_type) { case 'string': { - const ref = OutputReference.parse(request_body as string) + const ref = OutputReference.parse(request as string) if (ref !== undefined) { variables.add(ref) } break } case 'object': { - if (Array.isArray(request_body)) { - for (const value of request_body) { - StoryEvaluator.#extract_request_body_variables(value, variables) + if (Array.isArray(request)) { + for (const value of request) { + StoryEvaluator.#extract_request_variables(value, variables) } } else { - for (const [, value] of Object.entries(request_body as Record)) { - StoryEvaluator.#extract_request_body_variables(value, variables) + for (const [, value] of Object.entries(request as Record)) { + StoryEvaluator.#extract_request_variables(value, variables) } } break diff --git a/tools/src/tester/_generate_story_types.ts b/tools/src/tester/_generate_story_types.ts index d4ab6271e..d520fc02d 100644 --- a/tools/src/tester/_generate_story_types.ts +++ b/tools/src/tester/_generate_story_types.ts @@ -28,7 +28,7 @@ void js2ts.compile(schema, 'Story', * This file was automatically generated by json-schema-to-typescript. * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, * and run json-schema-to-typescript to regenerate this file by running: -* "npx ts-node tools/src/tester/_generate_story_types.ts" in a terminal. +* "npm run generate-types" in a terminal. */ ` }) diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index a89352eb0..a95104c50 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -39,7 +39,7 @@ export interface ChapterEvaluation { path?: string, request?: { parameters?: Record - request_body?: Evaluation + request?: Evaluation } response?: { status: Evaluation diff --git a/tools/src/tester/types/spec.types.ts b/tools/src/tester/types/spec.types.ts index 06689a9eb..6a6f0f10c 100644 --- a/tools/src/tester/types/spec.types.ts +++ b/tools/src/tester/types/spec.types.ts @@ -11,11 +11,11 @@ import { type OpenAPIV3 } from 'openapi-types' export type ParsedOperation = OpenAPIV3.OperationObject & { parameters: Record - requestBody: ParsedRequestBody + requestBody: ParsedRequest responses: Record } -export type ParsedRequestBody = OpenAPIV3.RequestBodyObject & { +export type ParsedRequest = OpenAPIV3.RequestBodyObject & { content: Record } diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 3b7fbe18b..e418ff762 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -14,7 +14,7 @@ * This file was automatically generated by json-schema-to-typescript. * DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file, * and run json-schema-to-typescript to regenerate this file by running: - * "npx ts-node tools/src/tester/_generate_story_types.ts" in a terminal. + * "npm run generate-types" in a terminal. */ /** @@ -32,6 +32,11 @@ export type SupplementalChapter = ChapterRequest & { * via the `definition` "Parameter". */ export type Parameter = (string | number | boolean)[] | string | number | boolean; +/** + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Header". + */ +export type Header = string | number | boolean; /** * This interface was referenced by `Story`'s JSON-Schema * via the `definition` "Payload". @@ -104,18 +109,21 @@ export interface ChapterRequest { parameters?: { [k: string]: Parameter; }; - request_body?: RequestBody; + request?: Request; output?: Output; version?: Version; retry?: Retry; } /** * This interface was referenced by `Story`'s JSON-Schema - * via the `definition` "RequestBody". + * via the `definition` "Request". */ -export interface RequestBody { +export interface Request { content_type?: string; - payload: Payload; + headers?: { + [k: string]: Header; + }; + payload?: Payload; } /** * Describes output for a chapter. diff --git a/tools/tests/linter/Operation.test.ts b/tools/tests/linter/Operation.test.ts index c5154eb0a..4634ba4cc 100644 --- a/tools/tests/linter/Operation.test.ts +++ b/tools/tests/linter/Operation.test.ts @@ -78,15 +78,15 @@ test('validate_description()', () => { test('validate_requestBody()', () => { const no_body = operation({ 'x-operation-group': 'indices.create' }) - expect(no_body.validate_request_body()) + expect(no_body.validate_request()) .toBeUndefined() const valid_body = operation({ 'x-operation-group': 'indices.create', requestBody: { $ref: '#/components/requestBodies/indices.create' } }) - expect(valid_body.validate_request_body()) + expect(valid_body.validate_request()) .toBeUndefined() const invalid_body = operation({ 'x-operation-group': 'indices.create', requestBody: { $ref: '#/components/requestBodies/indices.create.1' } }) - expect(invalid_body.validate_request_body()) + expect(invalid_body.validate_request()) .toEqual(invalid_body.error('The requestBody must be a reference object to \'#/components/requestBodies/indices.create\'.')) }) diff --git a/tools/tests/linter/OperationGroup.test.ts b/tools/tests/linter/OperationGroup.test.ts index 9ad7c1f65..92351abf2 100644 --- a/tools/tests/linter/OperationGroup.test.ts +++ b/tools/tests/linter/OperationGroup.test.ts @@ -45,13 +45,13 @@ test('validate_requestBody()', () => { const valid_request_bodies = operation_group([ { requestBody: { $ref: '#/components/requestBodies/indices.create' } }, { requestBody: { $ref: '#/components/requestBodies/indices.create' } }]) - expect(valid_request_bodies.validate_request_body()) + expect(valid_request_bodies.validate_request()) .toBeUndefined() const invalid_request_bodies = operation_group([ { requestBody: { $ref: '#/components/requestBodies/indices.create' } }, {}]) - expect(invalid_request_bodies.validate_request_body()) + expect(invalid_request_bodies.validate_request()) .toEqual(invalid_request_bodies.error('2 \'indices.create\' operations must have identical requestBody property.')) }) diff --git a/tools/tests/linter/factories/operation.ts b/tools/tests/linter/factories/operation.ts index b4f90363e..2a87677be 100644 --- a/tools/tests/linter/factories/operation.ts +++ b/tools/tests/linter/factories/operation.ts @@ -40,7 +40,7 @@ export function mocked_operation (returned_values: MockedReturnedValues): Operat op.validate_namespace = jest.fn() op.validate_operation_id = jest.fn() op.validate_description = jest.fn() - op.validate_request_body = jest.fn() + op.validate_request = jest.fn() op.validate_responses = jest.fn() op.validate_parameters = jest.fn() op.validate_path_parameters = jest.fn() @@ -49,7 +49,7 @@ export function mocked_operation (returned_values: MockedReturnedValues): Operat if (returned_values.validate_namespace != null) (op.validate_namespace as jest.Mock).mockReturnValue(returned_values.validate_namespace) if (returned_values.validate_operationId != null) (op.validate_operation_id as jest.Mock).mockReturnValue(returned_values.validate_operationId) if (returned_values.validate_description != null) (op.validate_description as jest.Mock).mockReturnValue(returned_values.validate_description) - if (returned_values.validate_requestBody != null) (op.validate_request_body as jest.Mock).mockReturnValue(returned_values.validate_requestBody) + if (returned_values.validate_requestBody != null) (op.validate_request as jest.Mock).mockReturnValue(returned_values.validate_requestBody) if (returned_values.validate_responses != null) (op.validate_responses as jest.Mock).mockReturnValue(returned_values.validate_responses) if (returned_values.validate_parameters != null) (op.validate_parameters as jest.Mock).mockReturnValue(returned_values.validate_parameters) if (returned_values.validate_path_parameters != null) (op.validate_path_parameters as jest.Mock).mockReturnValue(returned_values.validate_path_parameters) diff --git a/tools/tests/linter/factories/operation_group.ts b/tools/tests/linter/factories/operation_group.ts index 1e5fa04c6..ab68e3ea5 100644 --- a/tools/tests/linter/factories/operation_group.ts +++ b/tools/tests/linter/factories/operation_group.ts @@ -38,13 +38,13 @@ export function mocked_operation_group (returned_values: MockedReturnedValues, o op_group.validate_description = jest.fn() op_group.validate_external_docs = jest.fn() - op_group.validate_request_body = jest.fn() + op_group.validate_request = jest.fn() op_group.validate_responses = jest.fn() op_group.validate_query_parameters = jest.fn() if (returned_values.validate_description != null) (op_group.validate_description as jest.Mock).mockReturnValue(returned_values.validate_description) if (returned_values.validate_externalDocs != null) (op_group.validate_external_docs as jest.Mock).mockReturnValue(returned_values.validate_externalDocs) - if (returned_values.validate_requestBody != null) (op_group.validate_request_body as jest.Mock).mockReturnValue(returned_values.validate_requestBody) + if (returned_values.validate_requestBody != null) (op_group.validate_request as jest.Mock).mockReturnValue(returned_values.validate_requestBody) if (returned_values.validate_responses != null) (op_group.validate_responses as jest.Mock).mockReturnValue(returned_values.validate_responses) if (returned_values.validate_query_parameters != null) (op_group.validate_query_parameters as jest.Mock).mockReturnValue(returned_values.validate_query_parameters) diff --git a/tools/tests/tester/ChapterReader.test.ts b/tools/tests/tester/ChapterReader.test.ts index 34eb6e893..7e1776bb2 100644 --- a/tools/tests/tester/ChapterReader.test.ts +++ b/tools/tests/tester/ChapterReader.test.ts @@ -44,11 +44,11 @@ describe('ChapterReader', () => { path: 'path', method: 'GET', parameters: undefined, - request_body: undefined, + request: undefined, output: undefined }, new StoryOutputs()) - expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(result).toStrictEqual({ status: 200, content_type: 'application/json' }) expect(mocked_axios.request.mock.calls).toEqual([ [{ url: 'path', @@ -67,19 +67,19 @@ describe('ChapterReader', () => { path: '{index}/path', method: 'GET', parameters: { index: 'books' }, - request_body: undefined, + request: undefined, output: undefined }, new StoryOutputs()) - expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(result).toEqual({ status: 200, content_type: 'application/json' }) expect(mocked_axios.request.mock.calls).toEqual([ [{ url: 'books/path', method: 'GET', + data: undefined, headers: { 'Content-Type': 'application/json' }, params: {}, - paramsSerializer: expect.any(Function), - data: undefined + paramsSerializer: expect.any(Function) }] ]) }) @@ -90,11 +90,11 @@ describe('ChapterReader', () => { path: '/path', method: 'GET', parameters: { indexes: ['book1', 'book2'] }, - request_body: undefined, + request: undefined, output: undefined }, new StoryOutputs()) - expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(result).toEqual({ status: 200, content_type: 'application/json' }) expect(mocked_axios.request.mock.calls).toEqual([ [{ url: '/path', @@ -116,11 +116,11 @@ describe('ChapterReader', () => { path: 'path', method: 'POST', parameters: { 'x': 1 }, - request_body: { payload: { "body": "present" } }, + request: { payload: { "body": "present" } }, output: undefined }, new StoryOutputs()) - expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(result).toEqual({ status: 200, content_type: 'application/json' }) expect(mocked_axios.request.mock.calls).toEqual([ [{ url: 'path', @@ -139,14 +139,14 @@ describe('ChapterReader', () => { path: 'path', method: 'POST', parameters: { 'x': 1 }, - request_body: { + request: { content_type: 'application/x-ndjson', payload: [{ "body": "present" }] }, output: undefined }, new StoryOutputs()) - expect(result).toEqual({ status: 200, content_type: 'application/json', payload: undefined }) + expect(result).toEqual({ status: 200, content_type: 'application/json' }) expect(mocked_axios.request.mock.calls).toEqual([ [{ url: 'path', @@ -158,6 +158,67 @@ describe('ChapterReader', () => { }] ]) }) + + it('sends headers', async () => { + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'GET', + request: { + headers: { + 'string': 'bar', + 'number': 1, + 'boolean': true + }, + }, + output: undefined + }, new StoryOutputs()) + + expect(result).toStrictEqual({ status: 200, content_type: 'application/json' }) + expect(mocked_axios.request.mock.calls).toStrictEqual([ + [{ + url: 'path', + method: 'GET', + data: undefined, + headers: { + 'Content-Type': 'application/json', + 'string': 'bar', + 'number': 1, + 'boolean': true + }, + params: {}, + paramsSerializer: expect.any(Function) + }] + ]) + }) + + it('overwrites case-insensitive content-type', async () => { + const result = await reader.read({ + id: 'id', + path: 'path', + method: 'GET', + request: { + headers: { + 'content-type': 'application/overwritten' + }, + }, + output: undefined + }, new StoryOutputs()) + + expect(result).toStrictEqual({ status: 200, content_type: 'application/json' }) + expect(mocked_axios.request.mock.calls).toStrictEqual([ + [{ + url: 'path', + method: 'GET', + data: undefined, + headers: { + 'Content-Type': 'application/overwritten', + }, + params: {}, + paramsSerializer: expect.any(Function) + }] + ]) + }) }) describe('deserialize_payload', () => { diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index 775bffb2e..450d9c5a3 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -26,7 +26,7 @@ chapters: path: DELETE /{index} request: parameters: {} - request_body: + request: result: PASSED response: status: diff --git a/tools/tests/tester/fixtures/evals/error/output_error.yaml b/tools/tests/tester/fixtures/evals/error/output_error.yaml index ed4e57ac3..27ac940d3 100644 --- a/tools/tests/tester/fixtures/evals/error/output_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/output_error.yaml @@ -16,7 +16,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: payload_body: diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml index 91aeca18b..3f59fe0e8 100644 --- a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml @@ -14,7 +14,7 @@ chapters: index: result: FAILED message: data must be string - request_body: + request: result: PASSED response: status: @@ -33,7 +33,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: FAILED message: 'data contains unsupported properties: aliases' response: @@ -55,7 +55,7 @@ chapters: result: PASSED index: result: PASSED - request_body: + request: result: PASSED response: status: @@ -75,7 +75,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: PASSED response: status: @@ -96,7 +96,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: PASSED response: status: diff --git a/tools/tests/tester/fixtures/evals/failed/not_found.yaml b/tools/tests/tester/fixtures/evals/failed/not_found.yaml index ec760c0b1..ecbd88992 100644 --- a/tools/tests/tester/fixtures/evals/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/evals/failed/not_found.yaml @@ -25,7 +25,7 @@ chapters: timeout: result: FAILED message: Schema for "timeout" parameter not found. - request_body: + request: result: PASSED response: status: @@ -44,7 +44,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: FAILED message: Schema for "application/json" request body not found in the spec. response: @@ -64,7 +64,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: PASSED response: status: diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index f0118f111..bb74087e8 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -13,7 +13,7 @@ chapters: parameters: index: result: PASSED - request_body: + request: result: PASSED response: status: @@ -30,7 +30,24 @@ chapters: path: GET /_cat request: parameters: {} - request_body: + request: + result: PASSED + response: + status: + result: PASSED + payload_body: + result: PASSED + payload_schema: + result: PASSED + output_values: + result: SKIPPED + - title: This GET /_cat chapter with a header should pass. + overall: + result: PASSED + path: GET /_cat + request: + parameters: {} + request: result: PASSED response: status: @@ -49,7 +66,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: @@ -68,7 +85,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: @@ -87,7 +104,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: @@ -106,7 +123,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: @@ -125,7 +142,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: @@ -144,7 +161,7 @@ chapters: parameters: format: result: PASSED - request_body: + request: result: PASSED response: status: diff --git a/tools/tests/tester/fixtures/fail_non_existent_variable.yaml b/tools/tests/tester/fixtures/fail_non_existent_variable.yaml index cb4e80d61..3dc1a8c9d 100644 --- a/tools/tests/tester/fixtures/fail_non_existent_variable.yaml +++ b/tools/tests/tester/fixtures/fail_non_existent_variable.yaml @@ -7,7 +7,7 @@ chapters: id: create_model_group path: /_plugins/_ml/model_groups/_register method: POST - request_body: + request: payload: name: NLP_Group description: Model group for NLP models diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml index ba84d4200..45b00ee0b 100644 --- a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml @@ -19,7 +19,7 @@ chapters: method: PUT parameters: index: books - request_body: + request: payload: aliases: {} - synopsis: This chapter should fail because the response content type does not match. diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/failed/not_found.yaml index 0c0778268..0b61b03a9 100644 --- a/tools/tests/tester/fixtures/stories/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/stories/failed/not_found.yaml @@ -20,7 +20,7 @@ chapters: method: HEAD parameters: index: books - request_body: + request: payload: {} - synopsis: This chapter should fail because the response is not defined in the spec. path: /{index} diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml index b1d458f25..b4b08b963 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/passed.yaml @@ -21,6 +21,17 @@ chapters: response: status: 200 content_type: text/plain + - synopsis: This GET /_cat chapter with a header should pass. + path: /_cat + method: GET + warnings: + multiple-paths-detected: false + request: + headers: + User-Agent: OpenSearch API Spec/1.0 + response: + status: 200 + content_type: text/plain - synopsis: This GET /_cat/health chapter returns application/json and should pass. path: /_cat/health parameters: diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 57cbcff16..5ee331a14 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -97,7 +97,7 @@ export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation { if (c.request !== undefined) { result.request = { - request_body: flatten(c.request.request_body) + request: flatten(c.request.request) } if (c.request.parameters !== undefined) { diff --git a/tools/tests/tester/test.test.ts b/tools/tests/tester/test.test.ts index a7db537d7..ee76c19b9 100644 --- a/tools/tests/tester/test.test.ts +++ b/tools/tests/tester/test.test.ts @@ -10,7 +10,7 @@ import { spawnSync } from 'child_process' import * as ansi from 'tester/Ansi' import * as path from 'path' -import { type Chapter, type ChapterRequest, type Output, type RequestBody, type ActualResponse, Story } from 'tester/types/story.types' +import { type Chapter, type ChapterRequest, type Output, type Request, type ActualResponse, Story } from 'tester/types/story.types' import { type EvaluationWithOutput, Result, ChapterEvaluation, StoryEvaluation } from 'tester/types/eval.types' import { ChapterOutput } from 'tester/ChapterOutput' import StoryEvaluator from 'tester/StoryEvaluator' @@ -102,11 +102,11 @@ function dummy_chapter_request(id?: string, output?: Output): ChapterRequest { } } -function dummy_chapter_request_with_input(parameters?: Record, request_body?: RequestBody, id?: string, output?: Output): ChapterRequest { +function dummy_chapter_request_with_input(parameters?: Record, request?: Request, id?: string, output?: Output): ChapterRequest { return { ...dummy_chapter_request(id, output), parameters, - request_body + request } } From 10619725989a6b614dff93f9d80a8db55d0143d8 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Tue, 6 Aug 2024 00:59:35 +1200 Subject: [PATCH 04/31] Add additionalProperties to ErrorCause and fix creation_date type for list_dangling_indices (#462) Signed-off-by: Thomas Farr --- CHANGELOG.md | 4 +++- spec/schemas/_common.yaml | 4 ++++ spec/schemas/dangling_indices.list_dangling_indices.yaml | 2 ++ tools/src/_utils/index.ts | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b4f8c1..2f65144d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -62,7 +62,9 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_ppl`, `explain` and `stats` ([#460](https://github.com/opensearch-project/opensearch-api-specification/pull/460)) - Added tests against OpenSearch 3.0 ([#459](https://github.com/opensearch-project/opensearch-api-specification/pull/459)) - Added support for request headers in tests [#461](https://github.com/opensearch-project/opensearch-api-specification/pull/461) - +- Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) +- Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) + ### Changed - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189)) diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index afb4b6d1a..1c20ad16b 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -139,6 +139,7 @@ components: UnitMillis: description: Time unit for milliseconds. type: number + format: int64 DurationLarge: description: |- A date histogram interval. Similar to `Duration` with additional units: `w` (week), `M` (month), `q` (quarter) and `y` (year). @@ -314,6 +315,9 @@ components: $ref: '#/components/schemas/ErrorCause' required: - type + additionalProperties: + title: metadata + description: Additional details about the error. DurationValueUnitNanos: allOf: - $ref: '#/components/schemas/UnitNanos' diff --git a/spec/schemas/dangling_indices.list_dangling_indices.yaml b/spec/schemas/dangling_indices.list_dangling_indices.yaml index b9775aa58..a9f3a42cd 100644 --- a/spec/schemas/dangling_indices.list_dangling_indices.yaml +++ b/spec/schemas/dangling_indices.list_dangling_indices.yaml @@ -13,6 +13,8 @@ components: type: string index_uuid: type: string + creation_date: + $ref: '_common.yaml#/components/schemas/DateTime' creation_date_millis: $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' node_ids: diff --git a/tools/src/_utils/index.ts b/tools/src/_utils/index.ts index 3278e14ca..5177fd86e 100644 --- a/tools/src/_utils/index.ts +++ b/tools/src/_utils/index.ts @@ -53,7 +53,7 @@ export function determine_possible_schema_types (doc: OpenAPIV3.Document, schema if (schema?.anyOf !== undefined) return collect_all(schema.anyOf) if (schema?.oneOf !== undefined) return collect_all(schema.oneOf) - if (schema == null || Object.keys(schema).filter(k => k !== 'description').length == 0) return SCHEMA_OBJECT_TYPES + if (schema == null || Object.keys(schema).filter(k => k !== 'description' && k !== 'title').length == 0) return SCHEMA_OBJECT_TYPES throw new Error(`Unable to determine possible types of schema: ${to_json(schema)}`) } From c0d4a8581eb5f88d6e430ec72415db50b5645b62 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Mon, 5 Aug 2024 11:57:23 -0400 Subject: [PATCH 05/31] Added rest_total_hits_as_int test for _search (#466) * Added rest_total_hits_as_int test for _search Signed-off-by: Andy Wick * change request_body to request Signed-off-by: Andy Wick * Not gonna fight the linter Signed-off-by: Andy Wick * Add "Refn" to .cspell and back to test Signed-off-by: Andy Wick --------- Signed-off-by: Andy Wick --- .cspell | 3 +- .../_core/search/rest_total_hits_as_int.yaml | 125 ++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/_core/search/rest_total_hits_as_int.yaml diff --git a/.cspell b/.cspell index 80670e9b4..080c3937b 100644 --- a/.cspell +++ b/.cspell @@ -183,4 +183,5 @@ urldecode vectory whoamiprotected wordnet -Yrtsd \ No newline at end of file +Yrtsd +Refn diff --git a/tests/_core/search/rest_total_hits_as_int.yaml b/tests/_core/search/rest_total_hits_as_int.yaml new file mode 100644 index 000000000..1885bc282 --- /dev/null +++ b/tests/_core/search/rest_total_hits_as_int.yaml @@ -0,0 +1,125 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test search endpoint with rest_total_hits_as_int. +prologues: + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + director: Bennett Miller + title: Moneyball + year: 2011 + status: [201] + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + director: Nicolas Winding Refn + title: Drive + year: 2011 + status: [201] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search with rest_total_hits_as_int=true. + path: /{index}/_search + parameters: + index: movies + rest_total_hits_as_int: true + method: POST + request: + payload: + query: + match_all: {} + response: + status: 200 + payload: + timed_out: false + hits: + total: 2 + max_score: 1 + hits: + - _index: movies + _score: 1 + _source: + director: Bennett Miller + title: Moneyball + year: 2011 + - _index: movies + _score: 1 + _source: + director: Nicolas Winding Refn + title: Drive + year: 2011 + - synopsis: Search with rest_total_hits_as_int=false. + path: /{index}/_search + parameters: + index: movies + rest_total_hits_as_int: false + method: POST + request: + payload: + query: + match_all: {} + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 2 + relation: eq + max_score: 1 + hits: + - _index: movies + _score: 1 + _source: + director: Bennett Miller + title: Moneyball + year: 2011 + - _index: movies + _score: 1 + _source: + director: Nicolas Winding Refn + title: Drive + year: 2011 + - synopsis: Search with rest_total_hits_as_int=false track_total_hits=1. + path: /{index}/_search + parameters: + index: movies + rest_total_hits_as_int: false + method: POST + request: + payload: + track_total_hits: 1 + query: + match_all: {} + response: + status: 200 + payload: + timed_out: false + hits: + total: + value: 1 + relation: gte + max_score: 1 + hits: + - _index: movies + _score: 1 + _source: + director: Bennett Miller + title: Moneyball + year: 2011 + - _index: movies + _score: 1 + _source: + director: Nicolas Winding Refn + title: Drive + year: 2011 + From 21bd1068c058c1fac9d008754e49b72ab189cdce Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Mon, 5 Aug 2024 16:16:58 -0400 Subject: [PATCH 06/31] Add create-index blocked solution and switch to npm ci (#465) * Add create-index blocked solution and switch to npm ci Signed-off-by: Andy Wick * Updated CHANGELOG Signed-off-by: Andy Wick * add TOC and replace `npm install` with `npm ci` Signed-off-by: Andy Wick --------- Signed-off-by: Andy Wick --- .github/workflows/analyze-pr-changes.yml | 8 ++++---- .github/workflows/build.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test-spec.yml | 2 +- .github/workflows/test-tools-integ.yml | 4 ++-- .github/workflows/test-tools-unit.yml | 4 ++-- .github/workflows/validate-spec.yml | 2 +- CHANGELOG.md | 1 + DEVELOPER_GUIDE.md | 4 ++-- TESTING_GUIDE.md | 26 +++++++++++++++++++++++- 10 files changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/analyze-pr-changes.yml b/.github/workflows/analyze-pr-changes.yml index 14bbb1ed2..d962fdc9c 100644 --- a/.github/workflows/analyze-pr-changes.yml +++ b/.github/workflows/analyze-pr-changes.yml @@ -57,7 +57,7 @@ jobs: -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \ opensearch-with-api-plugin - npm install + npm ci npm run dump-cluster-spec -- --opensearch-insecure --output $CLUSTER_SPEC @@ -72,7 +72,7 @@ jobs: - name: Build BEFORE Spec shell: bash -eo pipefail {0} run: | - npm install + npm ci npm run merge -- --source ./spec --output $BEFORE_SPEC - name: Checkout AFTER Spec @@ -82,7 +82,7 @@ jobs: - name: Build AFTER Spec shell: bash -eo pipefail {0} run: | - npm install + npm ci npm run merge -- --source ./spec --output $AFTER_SPEC - name: Calculate Coverage @@ -216,4 +216,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: pr-comment - path: pr-comment.json \ No newline at end of file + path: pr-comment.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed72ed331..4caefaf13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,7 @@ jobs: node-version: '20' - name: Build - run: npm install && npm run merge + run: npm ci && npm run merge - name: Extract Branch Name id: branch diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 246c9edf9..b04c3d121 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,7 +25,7 @@ jobs: node-version: '20' - name: Install Dependencies - run: npm install + run: npm ci - name: Lint run: npm run lint diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 80f7577f8..2449a9b10 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -55,7 +55,7 @@ jobs: node-version: '20' - name: Install Dependencies - run: npm install + run: npm ci - name: Run OpenSearch Cluster working-directory: .github/opensearch-cluster diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index a48dd7518..915e840cd 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -43,7 +43,7 @@ jobs: node-version: '20' - name: Install Dependencies - run: npm install + run: npm ci - name: Tests run: | @@ -52,4 +52,4 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v4.0.1 with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/test-tools-unit.yml b/.github/workflows/test-tools-unit.yml index 00b585bb2..d18fc2525 100644 --- a/.github/workflows/test-tools-unit.yml +++ b/.github/workflows/test-tools-unit.yml @@ -33,7 +33,7 @@ jobs: node-version: '20' - name: Install Dependencies - run: npm install + run: npm ci - name: Tests run: | @@ -42,4 +42,4 @@ jobs: - name: Upload coverage to Codecov uses: codecov/codecov-action@v4.0.1 with: - token: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/validate-spec.yml b/.github/workflows/validate-spec.yml index 89213ac67..71f6f478d 100644 --- a/.github/workflows/validate-spec.yml +++ b/.github/workflows/validate-spec.yml @@ -25,7 +25,7 @@ jobs: node-version: '20' - name: Install Dependencies - run: npm install + run: npm ci - name: Lint Spec run: npm run lint:spec diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f65144d0..1a28aadc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for request headers in tests [#461](https://github.com/opensearch-project/opensearch-api-specification/pull/461) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) +- Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) ### Changed diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index b965e50a1..951f00cdd 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -159,7 +159,7 @@ A number of [tools](tools) have been authored using TypeScript to aid in the dev To be able to use or develop the tools, some setup is required: 1. Install [Node.js](https://nodejs.org/en/learn/getting-started/how-to-install-nodejs). -2. Run `npm install` from the repository's root. +2. Run `npm ci` from the repository's root. ### [Spec Merger](tools/src/merger) @@ -360,4 +360,4 @@ This workflow runs on PRs to invoke the [tools' integration tests](tools/tests) ### [Validate Spec](.github/workflows/validate-spec.yml) -This workflow runs on PRs to invoke the [spec linter](#spec-linter) and ensure the multi-file spec is correct and follows the design guidelines. \ No newline at end of file +This workflow runs on PRs to invoke the [spec linter](#spec-linter) and ensure the multi-file spec is correct and follows the design guidelines. diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 31323efcf..ad79f85d6 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,6 +1,9 @@ - [Spec Testing Guide](#spec-testing-guide) - [Running Spec Tests Locally](#running-spec-tests-locally) + - [Common Errors](#common-errors) + - [401 Unauthorized](#401-unauthorized) + - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) - [Writing Spec Tests](#writing-spec-tests) - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) @@ -41,7 +44,28 @@ Verbose output: npm run test:spec -- --opensearch-insecure --verbose ``` -Note: Remember to set the `OPENSEARCH_PASSWORD` environment variable everytime you start a new shell to run the tests. Failing to do so will result in 401 Unauthorized errors. +### Common Errors + +#### 401 Unauthorized + +Remember to set the `OPENSEARCH_PASSWORD` environment variable everytime you start a new shell to run the tests. + +#### FORBIDDEN/10/cluster create-index blocked (api) + +The cluster is most likely hitting a disk watermark threshold. This example sets the disk watermark thresholds to 1500MB low, 100MB high, and 500MB flood stage, allowing the cluster to create indices even if the disk is almost full. + +```bash +curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_cluster/settings -H 'Content-Type: application/json' -d' +{ + "persistent": { + "cluster.routing.allocation.disk.watermark.low": "1500mb", + "cluster.routing.allocation.disk.watermark.high": "1000mb", + "cluster.routing.allocation.disk.watermark.flood_stage": "500mb", + "cluster.blocks.create_index" : null + } +} +' +``` ## Writing Spec Tests From c7dbaafc5ab684b0274d9695c5209f3f61401dcc Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 5 Aug 2024 16:41:41 -0400 Subject: [PATCH 07/31] Validate test story descriptions. (#467) * Refactored validate_description and title into a base class. Signed-off-by: dblock * Validate story descriptions. Signed-off-by: dblock * Use a pattern to validate story synopsis and description. Signed-off-by: dblock --------- Signed-off-by: dblock --- json_schemas/test_story.schema.yaml | 2 ++ tests/_core/reindex/pipeline.yaml | 2 +- tests/indices/aliases/put_alias.yaml | 2 +- tests/ingest/pipeline.yaml | 3 +-- tests/ml/model_groups.yaml | 5 ++--- tests/ml/models.yaml | 3 +-- tests/ppl/explain.yaml | 2 +- tests/ppl/query.yaml | 2 +- tests/sql/close.yaml | 2 +- tests/sql/explain.yaml | 2 +- tests/sql/query.yaml | 2 +- tools/src/linter/components/Info.ts | 13 ++----------- tools/src/linter/components/Operation.ts | 5 +---- tools/src/linter/components/Schema.ts | 5 +---- .../linter/components/base/ValidatorBase.ts | 18 ++++++++++++++++++ tools/tests/linter/NamespaceFile.test.ts | 8 ++++---- tools/tests/linter/Operation.test.ts | 2 +- tools/tests/linter/Schema.test.ts | 2 +- tools/tests/tester/StoryValidator.test.ts | 8 ++++++++ .../tester/fixtures/invalid_description.yaml | 8 ++++++++ tools/tests/tester/fixtures/invalid_story.yaml | 4 ++-- 21 files changed, 59 insertions(+), 41 deletions(-) create mode 100644 tools/tests/tester/fixtures/invalid_description.yaml diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 829eb5286..6d1916509 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -6,6 +6,7 @@ properties: type: string description: type: string + pattern: ^\p{Lu}[\s\S]*\.$ prologues: type: array items: @@ -32,6 +33,7 @@ definitions: synopsis: type: string description: A brief description of the chapter. + pattern: ^\p{Lu}[\s\S]*\.$ response: $ref: '#/definitions/ExpectedResponse' warnings: diff --git a/tests/_core/reindex/pipeline.yaml b/tests/_core/reindex/pipeline.yaml index fbb6e1065..cdcae8230 100644 --- a/tests/_core/reindex/pipeline.yaml +++ b/tests/_core/reindex/pipeline.yaml @@ -16,7 +16,7 @@ prologues: method: PUT request: payload: - description: | + description: |- Splits the `title`` field into a `words` list. Computes the length of the title field and stores it in a new `length` field. Removes the `year` field. diff --git a/tests/indices/aliases/put_alias.yaml b/tests/indices/aliases/put_alias.yaml index ea65d3c53..7a7099efa 100644 --- a/tests/indices/aliases/put_alias.yaml +++ b/tests/indices/aliases/put_alias.yaml @@ -63,7 +63,7 @@ chapters: payload: acknowledged: true - - synopsis: Retrieve aliases + - synopsis: Retrieve aliases. path: /{index}/_alias method: GET parameters: diff --git a/tests/ingest/pipeline.yaml b/tests/ingest/pipeline.yaml index 3319d7026..5fd90d3ca 100644 --- a/tests/ingest/pipeline.yaml +++ b/tests/ingest/pipeline.yaml @@ -1,7 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: | - Test the creation of an ingest pipeline with a text embedding processor. +description: Test the creation of an ingest pipeline with a text embedding processor. epilogues: - path: /_ingest/pipeline/books_pipeline method: DELETE diff --git a/tests/ml/model_groups.yaml b/tests/ml/model_groups.yaml index 778863820..0e5e68b18 100644 --- a/tests/ml/model_groups.yaml +++ b/tests/ml/model_groups.yaml @@ -1,7 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: | - Test the creation of model groups. +description: Test the creation of model groups. version: '>= 2.11' prologues: - path: /_cluster/settings @@ -28,7 +27,7 @@ chapters: request: payload: name: NLP_Group - description: Model group for NLP models + description: Model group for NLP models. response: status: 200 output: diff --git a/tests/ml/models.yaml b/tests/ml/models.yaml index f82d2ecda..ecf8c3561 100644 --- a/tests/ml/models.yaml +++ b/tests/ml/models.yaml @@ -1,7 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: | - Test the creation of models. +description: Test the creation of models. version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/ppl/explain.yaml b/tests/ppl/explain.yaml index 89421fb70..70df71d79 100644 --- a/tests/ppl/explain.yaml +++ b/tests/ppl/explain.yaml @@ -12,7 +12,7 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get explain of SQL Query + - synopsis: Get explain of SQL Query. path: /_plugins/_ppl/_explain method: POST request: diff --git a/tests/ppl/query.yaml b/tests/ppl/query.yaml index b9b3548be..a525a85a0 100644 --- a/tests/ppl/query.yaml +++ b/tests/ppl/query.yaml @@ -14,7 +14,7 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get PPL query + - synopsis: Get PPL query. path: /_plugins/_ppl method: POST request: diff --git a/tests/sql/close.yaml b/tests/sql/close.yaml index 0f70d62c8..906ff98be 100644 --- a/tests/sql/close.yaml +++ b/tests/sql/close.yaml @@ -1,6 +1,6 @@ $schema: ../../json_schemas/test_story.schema.yaml -description: Test to explicitly clear the cursor context +description: Test to explicitly clear the cursor context. prologues: - path: /{index} diff --git a/tests/sql/explain.yaml b/tests/sql/explain.yaml index 245d65d83..13322ace3 100644 --- a/tests/sql/explain.yaml +++ b/tests/sql/explain.yaml @@ -18,7 +18,7 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get explain of SQL Query + - synopsis: Get explain of SQL Query. path: /_plugins/_sql/_explain method: POST request: diff --git a/tests/sql/query.yaml b/tests/sql/query.yaml index ba1d34317..8db0cfd1e 100644 --- a/tests/sql/query.yaml +++ b/tests/sql/query.yaml @@ -20,7 +20,7 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get SQL query + - synopsis: Get SQL query. path: /_plugins/_sql method: POST parameters: diff --git a/tools/src/linter/components/Info.ts b/tools/src/linter/components/Info.ts index 0ec39beca..c472cf7f1 100644 --- a/tools/src/linter/components/Info.ts +++ b/tools/src/linter/components/Info.ts @@ -10,9 +10,6 @@ import { type OpenAPIV3 } from 'openapi-types' import { type ValidationError } from 'types' import ValidatorBase from './base/ValidatorBase' -import { toLaxTitleCase } from 'titlecase' - -const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u export default class Info extends ValidatorBase { path: string @@ -32,16 +29,10 @@ export default class Info extends ValidatorBase { } validate_description (): ValidationError | undefined { - const description = this.spec?.description ?? '' - if (description === '') { return this.error('Missing description property.') } - if (!DESCRIPTION_REGEX.test(description)) { return this.error('Description must start with a capital letter and end with a period.') } + return this.validate_description_field(this.spec?.description, true) } validate_title (): ValidationError | undefined { - const title = this.spec?.title ?? '' - if (title === '') { return this.error('Missing title property.') } - const expected_title = toLaxTitleCase(title) - if (title.endsWith('.')) { return this.error('Title must not end with a period.') } - if (title !== expected_title) return this.error(`Title must be capitalized, expected '${expected_title}'.`) + return this.validate_title_field(this.spec?.title) } } diff --git a/tools/src/linter/components/Operation.ts b/tools/src/linter/components/Operation.ts index f55b29e27..2badcb6ac 100644 --- a/tools/src/linter/components/Operation.ts +++ b/tools/src/linter/components/Operation.ts @@ -12,7 +12,6 @@ import _ from 'lodash' import ValidatorBase from './base/ValidatorBase' const GROUP_REGEX = /^([a-z]+[a-z_]*[a-z]+\.)?([a-z]+[a-z_]*[a-z]+)$/ -const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u export default class Operation extends ValidatorBase { path: string @@ -65,9 +64,7 @@ export default class Operation extends ValidatorBase { } validate_description (): ValidationError | undefined { - const description = this.spec.description ?? '' - if (description === '') { return this.error('Missing description property.') } - if (!DESCRIPTION_REGEX.test(description)) return this.error('Description must start with a capital letter and end with a period.') + return this.validate_description_field(this.spec?.description, true) } validate_operation_id (): ValidationError | undefined { diff --git a/tools/src/linter/components/Schema.ts b/tools/src/linter/components/Schema.ts index a84438f92..2cdfd6a38 100644 --- a/tools/src/linter/components/Schema.ts +++ b/tools/src/linter/components/Schema.ts @@ -12,7 +12,6 @@ import { type OpenAPIV3 } from 'openapi-types' import { type ValidationError } from 'types' const NAME_REGEX = /^[A-Za-z0-9]+$/ -const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u export default class Schema extends ValidatorBase { name: string @@ -36,8 +35,6 @@ export default class Schema extends ValidatorBase { } validate_description (): ValidationError | undefined { - const description = this.spec.description ?? '' - if (description === '') return - if (!DESCRIPTION_REGEX.test(description)) { return this.error('Description must start with a capital letter and end with a period.') } + return this.validate_description_field(this.spec.description) } } diff --git a/tools/src/linter/components/base/ValidatorBase.ts b/tools/src/linter/components/base/ValidatorBase.ts index d92df8256..ae64fc066 100644 --- a/tools/src/linter/components/base/ValidatorBase.ts +++ b/tools/src/linter/components/base/ValidatorBase.ts @@ -7,7 +7,11 @@ * compatible open source license. */ +import { toLaxTitleCase } from 'titlecase' import { type ValidationError } from 'types' + +const DESCRIPTION_REGEX = /^\p{Lu}[\s\S]*\.$/u + export default class ValidatorBase { file: string location: string | undefined @@ -24,4 +28,18 @@ export default class ValidatorBase { validate (): ValidationError[] { throw new Error('Method not implemented.') } + + validate_description_field (value?: string, required: boolean = false, key: string = 'description'): ValidationError | undefined { + if (value === undefined) { return required ? this.error(`Missing ${key} property.`) : undefined } + if (value === '') { return this.error(`Empty ${key} property.`) } + if (!DESCRIPTION_REGEX.test(value)) { return this.error(`The ${key} must start with a capital letter and end with a period, got "${value}".`) } + } + + validate_title_field (value?: string, required: boolean = false, key: string = 'title'): ValidationError | undefined { + if (value === undefined) { return required ? this.error(`Missing ${key} property.`) : undefined } + if (value === '') { return this.error(`Empty ${key} property.`) } + const expected = toLaxTitleCase(value) + if (value.endsWith('.')) { return this.error(`The ${key} must not end with a period.`) } + if (value !== expected) return this.error(`The ${key} must be capitalized, expected "${expected}", not "${value}".`) + } } diff --git a/tools/tests/linter/NamespaceFile.test.ts b/tools/tests/linter/NamespaceFile.test.ts index 88cb22c91..51b6d995d 100644 --- a/tools/tests/linter/NamespaceFile.test.ts +++ b/tools/tests/linter/NamespaceFile.test.ts @@ -105,12 +105,12 @@ test('validate_info() periods', () => { { file: 'namespaces/invalid_info_periods.yaml', location: 'Info', - message: 'Title must not end with a period.' + message: 'The title must not end with a period.' }, { file: 'namespaces/invalid_info_periods.yaml', location: 'Info', - message: 'Description must start with a capital letter and end with a period.' + message: 'The description must start with a capital letter and end with a period, got "Description should have a period".' } ]) }) @@ -126,12 +126,12 @@ test('validate_info() capitals', () => { { file: 'namespaces/invalid_info_capitals.yaml', location: 'Info', - message: "Title must be capitalized, expected 'Title Must Be Capitalized'." + message: "The title must be capitalized, expected \"Title Must Be Capitalized\", not \"Title must be capitalized\"." }, { file: 'namespaces/invalid_info_capitals.yaml', location: 'Info', - message: "Description must start with a capital letter and end with a period." + message: "The description must start with a capital letter and end with a period, got \"description must start with a capital letter and end with a period.\"." } ]) }) diff --git a/tools/tests/linter/Operation.test.ts b/tools/tests/linter/Operation.test.ts index 4634ba4cc..e7fd124b6 100644 --- a/tools/tests/linter/Operation.test.ts +++ b/tools/tests/linter/Operation.test.ts @@ -69,7 +69,7 @@ test('validate_description()', () => { const invalid_description = operation({ 'x-operation-group': 'indices.create', description: 'This is a description without a period' }) expect(invalid_description.validate_description()) - .toEqual(invalid_description.error('Description must start with a capital letter and end with a period.')) + .toEqual(invalid_description.error('The description must start with a capital letter and end with a period, got "This is a description without a period".')) const valid_description = operation({ 'x-operation-group': 'indices.create', description: 'This is a description with a period.' }) expect(valid_description.validate_description()) diff --git a/tools/tests/linter/Schema.test.ts b/tools/tests/linter/Schema.test.ts index afedc8916..9c3f9cbfd 100644 --- a/tools/tests/linter/Schema.test.ts +++ b/tools/tests/linter/Schema.test.ts @@ -26,6 +26,6 @@ test('validate_description()', () => { expect(schema('Name', { description: 'Does not end with a period' }).validate_description()).toEqual({ file: '_common.yaml', location: '#/components/schemas/Name', - message: "Description must start with a capital letter and end with a period." + message: "The description must start with a capital letter and end with a period, got \"Does not end with a period\"." }) }) diff --git a/tools/tests/tester/StoryValidator.test.ts b/tools/tests/tester/StoryValidator.test.ts index 4a755b7a2..2d98159fa 100644 --- a/tools/tests/tester/StoryValidator.test.ts +++ b/tools/tests/tester/StoryValidator.test.ts @@ -35,6 +35,14 @@ describe('StoryValidator', () => { "data/chapters/1/method MUST be equal to one of the allowed values: GET, PUT, POST, DELETE, PATCH, HEAD, OPTIONS") }) + test('invalid description', () => { + const evaluation = validate('tools/tests/tester/fixtures/invalid_description.yaml') + expect(evaluation?.result).toBe('ERROR') + expect(evaluation?.message).toBe("Invalid Story: " + + "data/description must match pattern \"^\\p{Lu}[\\s\\S]*\\.$\" --- " + + "data/chapters/0/synopsis must match pattern \"^\\p{Lu}[\\s\\S]*\\.$\"") + }) + test('valid story', () => { const evaluation = validate('tools/tests/tester/fixtures/valid_story.yaml') expect(evaluation).toBeUndefined() diff --git a/tools/tests/tester/fixtures/invalid_description.yaml b/tools/tests/tester/fixtures/invalid_description.yaml new file mode 100644 index 000000000..eb4398fa1 --- /dev/null +++ b/tools/tests/tester/fixtures/invalid_description.yaml @@ -0,0 +1,8 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: This story description is missing a period + +chapters: + - synopsis: this synopsis is not capitalized. + method: GET + path: /_cat/health diff --git a/tools/tests/tester/fixtures/invalid_story.yaml b/tools/tests/tester/fixtures/invalid_story.yaml index b59730cd7..4e3af0817 100644 --- a/tools/tests/tester/fixtures/invalid_story.yaml +++ b/tools/tests/tester/fixtures/invalid_story.yaml @@ -2,9 +2,9 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: This story is invalid when validated against test_story.schema.yaml. chapters: - - synopsis: Missing Method + - synopsis: Missing method. path: /{index} - - synopsis: Invalid Method + - synopsis: Invalid method. path: / method: RETRIEVE epilogues: From 870a0183371876442b7a034c1266fc322ccc2cc1 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 6 Aug 2024 19:54:56 -0400 Subject: [PATCH 08/31] Allow response payload to reference variables. (#471) Signed-off-by: dblock --- CHANGELOG.md | 5 +- TESTING_GUIDE.md | 2 + .../nodes/plugins/index_state_management.yaml | 30 ++++ tools/src/tester/ChapterEvaluator.ts | 5 +- tools/src/tester/ChapterOutput.ts | 8 +- tools/src/tester/StoryOutputs.ts | 6 +- .../tester/SupplementalChapterEvaluator.ts | 7 +- tools/src/tester/test.ts | 2 +- tools/tests/tester/ChapterOutput.test.ts | 158 ++++++++++++++++++ tools/tests/tester/StoryOutputs.test.ts | 6 +- tools/tests/tester/helpers.ts | 2 +- tools/tests/tester/test.test.ts | 53 +----- 12 files changed, 218 insertions(+), 66 deletions(-) create mode 100644 tests/nodes/plugins/index_state_management.yaml create mode 100644 tools/tests/tester/ChapterOutput.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a28aadc4..1bebc33e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,7 +41,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395)) - Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395)) - Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396)) -- Add `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408)) +- Added `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408)) - Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409)) - Added support for skipping tests using semver range ([#410](https://github.com/opensearch-project/opensearch-api-specification/pull/410)) - Added `cluster_manager_timeout` to `HEAD /{index}` ([#421](https://github.com/opensearch-project/opensearch-api-specification/pull/421)) @@ -65,7 +65,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - +- Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) + ### Changed - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189)) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index ad79f85d6..a6d4456cf 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -167,6 +167,8 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) ``` As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group. +You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example. + ### Managing Versions It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. diff --git a/tests/nodes/plugins/index_state_management.yaml b/tests/nodes/plugins/index_state_management.yaml new file mode 100644 index 000000000..4f95866d6 --- /dev/null +++ b/tests/nodes/plugins/index_state_management.yaml @@ -0,0 +1,30 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Get index_state_management node info settings. +prologues: + - path: /_cat/nodes + id: node + method: GET + parameters: + full_id: true + size: 1 + format: json + h: id + output: + id: payload[0].id +chapters: + - synopsis: Get node info. + path: /_nodes/{node_id_or_metric} + method: GET + parameters: + node_id_or_metric: ${node.id} + response: + status: 200 + payload: + nodes: + ${node.id}: + settings: + plugins: + index_state_management: + job_interval: '1' + diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index e06fca445..0be147f4b 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -66,9 +66,12 @@ export default class ChapterEvaluator { const params = this.#evaluate_parameters(chapter, operation) const request = this.#evaluate_request(chapter, operation) const status = this.#evaluate_status(chapter, response) - const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED } const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED } const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } } + const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload + const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED } + + if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) const evaluations = _.compact(_.concat( Object.values(params), diff --git a/tools/src/tester/ChapterOutput.ts b/tools/src/tester/ChapterOutput.ts index 5a8e07b28..d8fd6d115 100644 --- a/tools/src/tester/ChapterOutput.ts +++ b/tools/src/tester/ChapterOutput.ts @@ -30,19 +30,17 @@ export class ChapterOutput { if (!output) return { evaluation: { result: Result.SKIPPED } } const chapter_output = new ChapterOutput({}) for (const [name, path] of Object.entries(output)) { - const [source, ...rest] = path.split('.') - const keys = rest.join('.') let value: any - if (source === 'payload') { + if (path == 'payload' || path.startsWith('payload.') || path.match(/^payload\[\d*\]/)) { if (response.payload === undefined) { return { evaluation: { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path } } } - value = keys.length === 0 ? response.payload : _.get(response.payload, keys) + value = _.get(response, path) if (value === undefined) { return { evaluation: { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` } } } } else { - return { evaluation: { result: Result.ERROR, message: 'Unknown output source: ' + source } } + return { evaluation: { result: Result.ERROR, message: `Unknown output source: ${path.split('.')[0]}.` } } } chapter_output.set(name, value) } diff --git a/tools/src/tester/StoryOutputs.ts b/tools/src/tester/StoryOutputs.ts index c55ac21d6..41b8d9e71 100644 --- a/tools/src/tester/StoryOutputs.ts +++ b/tools/src/tester/StoryOutputs.ts @@ -68,12 +68,14 @@ export class StoryOutputs { resolved_array.push(this.resolve_value(value)) } return resolved_array - } else { + } else if (payload !== null) { const resolved_obj: Record = {} for (const [key, value] of Object.entries(payload as Record)) { - resolved_obj[key] = this.resolve_value(value) + resolved_obj[this.resolve_value(key)] = this.resolve_value(value) } return resolved_obj + } else { + return payload } default: return payload diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index 81b680bef..e507e9915 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -14,12 +14,16 @@ import { StoryOutputs } from "./StoryOutputs"; import { overall_result } from "./helpers"; import { ChapterEvaluation, Result } from "./types/eval.types"; import { SupplementalChapter } from "./types/story.types"; +import { Logger } from "../Logger"; +import { to_json } from "../helpers"; export default class SupplementalChapterEvaluator { private readonly _chapter_reader: ChapterReader; + private readonly logger: Logger; - constructor(chapter_reader: ChapterReader) { + constructor(chapter_reader: ChapterReader, logger: Logger) { this._chapter_reader = chapter_reader; + this.logger = logger } async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<{ evaluation: ChapterEvaluation, evaluation_error: boolean }> { @@ -27,6 +31,7 @@ export default class SupplementalChapterEvaluator { const response = await this._chapter_reader.read(chapter, story_outputs) const status = chapter.status ?? [200, 201] const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output) + if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`) let response_evaluation: ChapterEvaluation const passed_evaluation = { title, overall: { result: Result.PASSED } } if (status.includes(response.status)) { diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 98a875e09..601d3ff5d 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -57,7 +57,7 @@ const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Lo const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) -const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) +const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) diff --git a/tools/tests/tester/ChapterOutput.test.ts b/tools/tests/tester/ChapterOutput.test.ts new file mode 100644 index 000000000..f5ace6ed7 --- /dev/null +++ b/tools/tests/tester/ChapterOutput.test.ts @@ -0,0 +1,158 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { ChapterOutput } from 'tester/ChapterOutput' +import { EvaluationWithOutput, Result } from 'tester/types/eval.types' +import { ActualResponse } from 'tester/types/story.types' + +function create_response(payload: any): ActualResponse { + return { + status: 200, + content_type: 'application/json', + payload + } +} + +function passed_output(output: Record): EvaluationWithOutput { + return { + evaluation: { result: Result.PASSED }, + output: new ChapterOutput(output) + } +} + +describe('with an object response', () => { + const response: ActualResponse = create_response({ + a: { + b: { + c: 1 + }, + arr: [ + { d: 2 }, + { e: 3 } + ] + } + }) + + test('returns nested values', () => { + const output = { + c: 'payload.a.b.c', + d: 'payload.a.arr[0].d', + e: 'payload.a.arr[1].e' + } + + expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({ + c: 1, + d: 2, + e: 3 + })) + }) + + test('extracts complete payload', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( + passed_output({ x: response.payload }) + ) + }) + + test('errors on undefined value', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload.a.b.x[0]`.' + } + }) + }) + + test('errors on invalid source', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Unknown output source: foobar.' + } + }) + }) +}) + +describe('with an array response', () => { + const response: ActualResponse = create_response([ + { + a: { + b: { + c: 1 + }, + arr: [ + { d: 2 }, + { e: 3 } + ] + } + },{ + a: { + b: { + c: 2 + }, + arr: [ + { d: 3 }, + { e: 4 } + ] + } + }, + ]) + + test('returns nested values', () => { + const output = { + c1: 'payload[0].a.b.c', + d1: 'payload[0].a.arr[0].d', + e1: 'payload[0].a.arr[1].e', + c2: 'payload[1].a.b.c', + d2: 'payload[1].a.arr[0].d', + e2: 'payload[1].a.arr[1].e' + } + + expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({ + c1: 1, + d1: 2, + e1: 3, + c2: 2, + d2: 3, + e2: 4 + })) + }) + + test('extracts complete payload', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( + passed_output({ x: response.payload }) + ) + }) + + test('errors on undefined value', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload[0].a.b.x[0]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload[0].a.b.x[0]`.' + } + }) + }) + + test('errors on invalid source', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Unknown output source: foobar.' + } + }) + }) + + test('errors on invalid index', () => { + expect(ChapterOutput.extract_output_values(response, { x: 'payload[2]' })).toEqual({ + evaluation: { + result: Result.ERROR, + message: 'Expected to find non undefined value at `payload[2]`.' + } + }) + }) +}) diff --git a/tools/tests/tester/StoryOutputs.test.ts b/tools/tests/tester/StoryOutputs.test.ts index 13d4a082a..3017b7a66 100644 --- a/tools/tests/tester/StoryOutputs.test.ts +++ b/tools/tests/tester/StoryOutputs.test.ts @@ -32,7 +32,8 @@ test('resolve_value', () => { e: 'str', f: true }, - g: 123 + g: 123, + '${chapter_id.x}': 345 } expect(story_outputs.resolve_value(value)).toEqual( { @@ -43,7 +44,8 @@ test('resolve_value', () => { e: 'str', f: true }, - g: 123 + g: 123, + 1: 345 } ) }) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 5ee331a14..dd6758d2a 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -48,7 +48,7 @@ export function construct_tester_components (spec_path: string): { const chapter_reader = new ChapterReader(opensearch_http_client, logger) const schema_validator = new SchemaValidator(specification, logger) const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger) - const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) + const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) const result_logger = new NoOpResultLogger() diff --git a/tools/tests/tester/test.test.ts b/tools/tests/tester/test.test.ts index ee76c19b9..e6677185d 100644 --- a/tools/tests/tester/test.test.ts +++ b/tools/tests/tester/test.test.ts @@ -10,9 +10,8 @@ import { spawnSync } from 'child_process' import * as ansi from 'tester/Ansi' import * as path from 'path' -import { type Chapter, type ChapterRequest, type Output, type Request, type ActualResponse, Story } from 'tester/types/story.types' -import { type EvaluationWithOutput, Result, ChapterEvaluation, StoryEvaluation } from 'tester/types/eval.types' -import { ChapterOutput } from 'tester/ChapterOutput' +import { type Chapter, type ChapterRequest, type Output, type Request, Story } from 'tester/types/story.types' +import { ChapterEvaluation, Result, StoryEvaluation } from 'tester/types/eval.types' import StoryEvaluator from 'tester/StoryEvaluator' const spec = (args: string[]): any => { @@ -45,54 +44,6 @@ test('invalid story', () => { ) }) -function create_response(payload: any): ActualResponse { - return { - status: 200, - content_type: 'application/json', - payload - } -} - -function passed_output(output: Record): EvaluationWithOutput { - return { - evaluation: { result: Result.PASSED }, - output: new ChapterOutput(output) - } -} - -test('extract_output_values', () => { - const response: ActualResponse = create_response({ - a: { - b: { - c: 1 - }, - arr: [ - { d: 2 }, - { e: 3 } - ] - } - }) - const output1 = { - c: 'payload.a.b.c', - d: 'payload.a.arr[0].d', - e: 'payload.a.arr[1].e' - } - expect(ChapterOutput.extract_output_values(response, output1)).toEqual(passed_output({ - c: 1, - d: 2, - e: 3 - })) - expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual( - passed_output({ x: response.payload }) - ) - expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({ - evaluation: { - result: Result.ERROR, - message: 'Expected to find non undefined value at `payload.a.b.x[0]`.' - } - }) -}) - function dummy_chapter_request(id?: string, output?: Output): ChapterRequest { return { id, From cfe357b7f066933991daab7ba4bf5840d9fd8546 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 6 Aug 2024 19:55:43 -0400 Subject: [PATCH 09/31] Increase memory allocated to OpenSearch in GHA. (#470) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 +- tests/sql/close.yaml | 6 ------ tests/sql/explain.yaml | 6 ------ tests/sql/query.yaml | 6 ------ 4 files changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 2449a9b10..0ec43c6e5 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -43,7 +43,7 @@ jobs: OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }} OPENSEARCH_VERSION: ${{ matrix.entry.version }} OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} - OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} + OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms2g -Xmx2g steps: - name: Checkout Repo diff --git a/tests/sql/close.yaml b/tests/sql/close.yaml index 906ff98be..b82b50be6 100644 --- a/tests/sql/close.yaml +++ b/tests/sql/close.yaml @@ -22,12 +22,6 @@ prologues: method: POST parameters: index: books - - path: _plugins/_query/settings - method: PUT - request: - payload: - transient: - plugins.query.memory_limit: 100% - id: sql_query path: /_plugins/_sql method: POST diff --git a/tests/sql/explain.yaml b/tests/sql/explain.yaml index 13322ace3..22a648f58 100644 --- a/tests/sql/explain.yaml +++ b/tests/sql/explain.yaml @@ -7,12 +7,6 @@ prologues: method: PUT request: payload: {} - - path: _plugins/_query/settings - method: PUT - request: - payload: - transient: - plugins.query.memory_limit: 100% epilogues: - path: /books method: DELETE diff --git a/tests/sql/query.yaml b/tests/sql/query.yaml index 8db0cfd1e..96bf4ee43 100644 --- a/tests/sql/query.yaml +++ b/tests/sql/query.yaml @@ -9,12 +9,6 @@ prologues: index: books request: payload: {} - - path: _plugins/_query/settings - method: PUT - request: - payload: - transient: - plugins.query.memory_limit: 100% epilogues: - path: /books method: DELETE From 41360b711c5ec35be0e2c4895b19d3efe2727fe6 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 7 Aug 2024 13:22:18 -0400 Subject: [PATCH 10/31] Split test suite with separate docker-compose. (#472) * Split test suite with separate docker-compose. Signed-off-by: dblock --- .github/workflows/test-spec.yml | 15 ++++++++----- .github/workflows/test-tools-integ.yml | 2 +- CHANGELOG.md | 1 + TESTING_GUIDE.md | 16 ++++++++------ tests/{ => default}/_core/bulk.yaml | 2 +- tests/{ => default}/_core/info.yaml | 2 +- tests/{ => default}/_core/mapping.yaml | 2 +- tests/{ => default}/_core/reindex.yaml | 2 +- .../{ => default}/_core/reindex/pipeline.yaml | 2 +- tests/{ => default}/_core/search/_source.yaml | 2 +- .../search/cancel_after_time_interval.yaml | 2 +- .../_core/search/geo_distance.yaml | 2 +- tests/{ => default}/_core/search/match.yaml | 2 +- .../_core/search/multi_match.yaml | 2 +- .../_core/search/phase_took.yaml | 2 +- .../_core/search/rest_total_hits_as_int.yaml | 2 +- .../_core/search/seq_no_primary_term.yaml | 2 +- tests/{ => default}/_core/search/size.yaml | 2 +- tests/{ => default}/cat/aliases.yaml | 2 +- tests/{ => default}/cat/allocation.yaml | 2 +- tests/{ => default}/cat/cluster_manager.yaml | 2 +- tests/{ => default}/cat/count.yaml | 2 +- tests/{ => default}/cat/fielddata.yaml | 2 +- tests/{ => default}/cat/health.yaml | 2 +- tests/{ => default}/cat/index.yaml | 2 +- tests/{ => default}/cat/indices.yaml | 2 +- tests/{ => default}/cat/master.yaml | 2 +- tests/{ => default}/cat/nodeattrs.yaml | 2 +- tests/{ => default}/cat/nodes.yaml | 2 +- tests/{ => default}/cat/pending_tasks.yaml | 2 +- tests/{ => default}/cat/pit_segments/all.yaml | 2 +- .../cat/pit_segments/pit_segments.yaml | 2 +- tests/{ => default}/cat/plugins.yaml | 2 +- tests/{ => default}/cat/recovery.yaml | 2 +- .../cat/segment_replication.yaml | 2 +- tests/{ => default}/cat/segments.yaml | 2 +- tests/{ => default}/cat/shards.yaml | 2 +- tests/{ => default}/cat/tasks.yaml | 2 +- tests/{ => default}/cat/templates.yaml | 2 +- tests/{ => default}/cat/thread_pool.yaml | 2 +- .../cluster/component_template.yaml | 2 +- tests/default/docker-compose.yml | 12 ++++++++++ tests/{ => default}/indices/alias/alias.yaml | 2 +- .../indices/aliases/aliases.yaml | 2 +- .../indices/aliases/put_alias.yaml | 2 +- tests/{ => default}/indices/cache.yaml | 2 +- tests/{ => default}/indices/clone.yaml | 2 +- tests/{ => default}/indices/close.yaml | 2 +- .../indices/component_template.yaml | 2 +- tests/{ => default}/indices/dangling.yaml | 2 +- .../indices/data_stream/data_stream.yaml | 2 +- .../indices/data_stream/rollover.yaml | 2 +- .../indices/data_stream/stats.yaml | 2 +- tests/{ => default}/indices/doc.yaml | 2 +- tests/{ => default}/indices/flush.yaml | 2 +- tests/{ => default}/indices/forcemerge.yaml | 2 +- tests/{ => default}/indices/index.yaml | 2 +- .../{ => default}/indices/index_template.yaml | 2 +- tests/{ => default}/indices/mapping.yml | 2 +- tests/{ => default}/indices/open.yaml | 2 +- tests/{ => default}/indices/refresh.yaml | 2 +- tests/{ => default}/indices/settings.yaml | 2 +- tests/{ => default}/indices/shrink.yaml | 2 +- tests/{ => default}/indices/split.yaml | 2 +- tests/{ => default}/indices/stats.yaml | 2 +- .../indices/update_by_query.yaml | 2 +- tests/{ => default}/ingest/pipeline.yaml | 2 +- tests/{ => default}/ml/model_groups.yaml | 2 +- tests/{ => default}/ml/models.yaml | 2 +- tests/{ => default}/nodes/info.yaml | 2 +- tests/{ => default}/nodes/stats.yaml | 2 +- tests/{ => default}/ppl/explain.yaml | 2 +- tests/{ => default}/ppl/query.yaml | 2 +- tests/{ => default}/ppl/stats.yaml | 2 +- tests/{ => default}/sql/close.yaml | 2 +- tests/{ => default}/sql/explain.yaml | 2 +- tests/{ => default}/sql/query.yaml | 2 +- tests/{ => default}/sql/settings.yaml | 2 +- tests/{ => default}/sql/stats.yaml | 2 +- .../docker-compose.yml | 1 - .../nodes/plugins/index_state_management.yaml | 2 +- tests/{ => snapshot}/cat/repositories.yaml | 2 +- tests/{ => snapshot}/cat/snapshots.yaml | 2 +- tests/snapshot/docker-compose.yml | 13 +++++++++++ tools/src/tester/TestRunner.ts | 22 +++++++++++++++---- tools/src/tester/test.ts | 2 +- tools/tests/tester/integ/TestRunner.test.ts | 10 +++++++++ 87 files changed, 152 insertions(+), 96 deletions(-) rename tests/{ => default}/_core/bulk.yaml (95%) rename tests/{ => default}/_core/info.yaml (88%) rename tests/{ => default}/_core/mapping.yaml (95%) rename tests/{ => default}/_core/reindex.yaml (97%) rename tests/{ => default}/_core/reindex/pipeline.yaml (97%) rename tests/{ => default}/_core/search/_source.yaml (98%) rename tests/{ => default}/_core/search/cancel_after_time_interval.yaml (90%) rename tests/{ => default}/_core/search/geo_distance.yaml (95%) rename tests/{ => default}/_core/search/match.yaml (95%) rename tests/{ => default}/_core/search/multi_match.yaml (94%) rename tests/{ => default}/_core/search/phase_took.yaml (90%) rename tests/{ => default}/_core/search/rest_total_hits_as_int.yaml (98%) rename tests/{ => default}/_core/search/seq_no_primary_term.yaml (93%) rename tests/{ => default}/_core/search/size.yaml (95%) rename tests/{ => default}/cat/aliases.yaml (93%) rename tests/{ => default}/cat/allocation.yaml (79%) rename tests/{ => default}/cat/cluster_manager.yaml (81%) rename tests/{ => default}/cat/count.yaml (78%) rename tests/{ => default}/cat/fielddata.yaml (89%) rename tests/{ => default}/cat/health.yaml (98%) rename tests/{ => default}/cat/index.yaml (76%) rename tests/{ => default}/cat/indices.yaml (97%) rename tests/{ => default}/cat/master.yaml (80%) rename tests/{ => default}/cat/nodeattrs.yaml (84%) rename tests/{ => default}/cat/nodes.yaml (89%) rename tests/{ => default}/cat/pending_tasks.yaml (79%) rename tests/{ => default}/cat/pit_segments/all.yaml (92%) rename tests/{ => default}/cat/pit_segments/pit_segments.yaml (93%) rename tests/{ => default}/cat/plugins.yaml (82%) rename tests/{ => default}/cat/recovery.yaml (78%) rename tests/{ => default}/cat/segment_replication.yaml (81%) rename tests/{ => default}/cat/segments.yaml (78%) rename tests/{ => default}/cat/shards.yaml (90%) rename tests/{ => default}/cat/tasks.yaml (78%) rename tests/{ => default}/cat/templates.yaml (93%) rename tests/{ => default}/cat/thread_pool.yaml (89%) rename tests/{ => default}/cluster/component_template.yaml (97%) create mode 100644 tests/default/docker-compose.yml rename tests/{ => default}/indices/alias/alias.yaml (96%) rename tests/{ => default}/indices/aliases/aliases.yaml (93%) rename tests/{ => default}/indices/aliases/put_alias.yaml (96%) rename tests/{ => default}/indices/cache.yaml (95%) rename tests/{ => default}/indices/clone.yaml (95%) rename tests/{ => default}/indices/close.yaml (95%) rename tests/{ => default}/indices/component_template.yaml (98%) rename tests/{ => default}/indices/dangling.yaml (68%) rename tests/{ => default}/indices/data_stream/data_stream.yaml (95%) rename tests/{ => default}/indices/data_stream/rollover.yaml (93%) rename tests/{ => default}/indices/data_stream/stats.yaml (93%) rename tests/{ => default}/indices/doc.yaml (95%) rename tests/{ => default}/indices/flush.yaml (94%) rename tests/{ => default}/indices/forcemerge.yaml (96%) rename tests/{ => default}/indices/index.yaml (97%) rename tests/{ => default}/indices/index_template.yaml (97%) rename tests/{ => default}/indices/mapping.yml (98%) rename tests/{ => default}/indices/open.yaml (96%) rename tests/{ => default}/indices/refresh.yaml (92%) rename tests/{ => default}/indices/settings.yaml (97%) rename tests/{ => default}/indices/shrink.yaml (97%) rename tests/{ => default}/indices/split.yaml (97%) rename tests/{ => default}/indices/stats.yaml (95%) rename tests/{ => default}/indices/update_by_query.yaml (96%) rename tests/{ => default}/ingest/pipeline.yaml (94%) rename tests/{ => default}/ml/model_groups.yaml (95%) rename tests/{ => default}/ml/models.yaml (94%) rename tests/{ => default}/nodes/info.yaml (84%) rename tests/{ => default}/nodes/stats.yaml (98%) rename tests/{ => default}/ppl/explain.yaml (88%) rename tests/{ => default}/ppl/query.yaml (88%) rename tests/{ => default}/ppl/stats.yaml (78%) rename tests/{ => default}/sql/close.yaml (95%) rename tests/{ => default}/sql/explain.yaml (88%) rename tests/{ => default}/sql/query.yaml (89%) rename tests/{ => default}/sql/settings.yaml (94%) rename tests/{ => default}/sql/stats.yaml (78%) rename {.github/opensearch-cluster => tests/plugins/index_state_management}/docker-compose.yml (92%) rename tests/{ => plugins/index_state_management}/nodes/plugins/index_state_management.yaml (90%) rename tests/{ => snapshot}/cat/repositories.yaml (92%) rename tests/{ => snapshot}/cat/snapshots.yaml (90%) create mode 100644 tests/snapshot/docker-compose.yml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 0ec43c6e5..3fa404483 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -28,6 +28,10 @@ jobs: - version: 2.0.0 admin_password: admin - version: 2.15.0 + - version: 2.15.0 + tests: plugins/index_state_management + - version: 2.15.0 + tests: snapshot - version: 2.16.0 hub: opensearchstaging ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e' @@ -35,7 +39,7 @@ jobs: hub: opensearchstaging ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b' - name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}) + name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest env: @@ -58,7 +62,7 @@ jobs: run: npm ci - name: Run OpenSearch Cluster - working-directory: .github/opensearch-cluster + working-directory: tests/${{ matrix.entry.tests || 'default' }} run: docker compose up -d - name: Get Container ID @@ -71,7 +75,8 @@ jobs: npm run test:spec -- \ --opensearch-insecure \ --opensearch-version=${{ matrix.entry.version }} \ - --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json + --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json \ + --tests=tests/${{ matrix.entry.tests || 'default' }} - name: Get Container Logs if: failure() && steps.container.outputs.CONTAINER_ID @@ -82,8 +87,8 @@ jobs: - name: Upload Test Coverage Results uses: actions/upload-artifact@v4 with: - name: coverage-${{ matrix.entry.version }} - path: coverage/test-spec-coverage-${{ matrix.entry.version }}.json + name: coverage-${{ matrix.entry.version }}-${{ hashFiles(format('tests/{0}', matrix.entry.tests || 'default')) }} + path: coverage/test-spec-coverage-${{ matrix.entry.version }}-${{ matrix.entry.tests || 'default' }}.json merge-coverage: runs-on: ubuntu-latest diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index 915e840cd..660a55e7e 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@v4 - name: Run OpenSearch Cluster - working-directory: .github/opensearch-cluster + working-directory: tests/default run: | docker compose up -d sleep 15 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bebc33e4..d0e0e18a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Tester tool now provides better context for non-2XX responses when --verbose is used ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359)) - Lock testing for next release of OpenSearch to a specific SHA ([#431](https://github.com/opensearch-project/opensearch-api-specification/pull/431)) - Replace nullable with null type ([#436](https://github.com/opensearch-project/opensearch-api-specification/pull/436)) +- Split test suite ([#472])(https://github.com/opensearch-project/opensearch-api-specification/pull/472) ### Deprecated diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index a6d4456cf..d50d4f5b8 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -25,7 +25,7 @@ Set up an OpenSearch cluster with Docker: (Replace `<>` with your desired password. If not provided, the default password inside the `docker-compose.yml` file will be used.) ```bash export OPENSEARCH_PASSWORD=<> -cd .github/opensearch-cluster +cd tests/default docker compose up -d ``` @@ -36,7 +36,7 @@ npm run test:spec -- --opensearch-insecure Run a specific test story: ```bash -npm run test:spec -- --opensearch-insecure --tests tests/_core/info.yaml +npm run test:spec -- --opensearch-insecure --tests tests/default/_core/info.yaml ``` Verbose output: @@ -69,7 +69,9 @@ curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_clu ## Writing Spec Tests -The spec tests reside in the [tests/](tests) directory. Tests are organized in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/indices/index.yaml](tests/indices/index.yaml) (for `/{index}`), and [tests/indices/doc.yaml](tests/indices/doc.yaml) (for `/{index}/_doc`). +The spec tests reside in the [tests/](tests) directory. Tests are organized in suites ([default](tests/default/), etc.), and subsequently in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/default/indices/index.yaml](tests/default/indices/index.yaml) (for `/{index}`), and [tests/default/indices/doc.yaml](tests/default/indices/doc.yaml) (for `/{index}/_doc`). + +Additional suites require custom configuration that is defined in a separate `docker-compose.yml`. For example [tests/plugins/index_state_management/docker-compose.yml](tests/plugins/index_state_management/docker-compose.yml) uses a custom setting of `plugins.index_state_management.job_interval=1` to cause the `/_nodes` API to return plugin information tested in [tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml](tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml). Each yaml file in the tests directory represents a test story that tests a collection of related operations. @@ -82,9 +84,9 @@ Check the [test_story JSON Schema](json_schemas/test_story.schema.yaml) for the ### Simple Test Story -Below is the simplified version of the test story that tests the [index operations](tests/indices/index.yaml): +Below is the simplified version of the test story that tests the [index operations](tests/default/indices/index.yaml): ```yaml -$schema: ../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly. +$schema: ../../json_schemas/test_story.schema.yaml # The schema of the test story. Include this line so that your editor can validate the test story on the fly. description: This story tests all endpoints relevant the lifecycle of an index, from creation to deletion. @@ -134,7 +136,7 @@ chapters: ### Using Output from Previous Chapters -Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) test story: +Consider the following chapters in [ml/model_groups](tests/default/ml/model_groups.yaml) test story: ```yaml - synopsis: Create model group. id: create_model_group # Only needed if you want to refer to this chapter in another chapter. @@ -167,7 +169,7 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml) ``` As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group. -You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example. +You can also reuse output in payload expectations. See [tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml](tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml) for an example. ### Managing Versions diff --git a/tests/_core/bulk.yaml b/tests/default/_core/bulk.yaml similarity index 95% rename from tests/_core/bulk.yaml rename to tests/default/_core/bulk.yaml index abcbb438f..3ed172528 100644 --- a/tests/_core/bulk.yaml +++ b/tests/default/_core/bulk.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test bulk endpoint. epilogues: diff --git a/tests/_core/info.yaml b/tests/default/_core/info.yaml similarity index 88% rename from tests/_core/info.yaml rename to tests/default/_core/info.yaml index 6635498bc..67e245235 100644 --- a/tests/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -1,5 +1,5 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. chapters: diff --git a/tests/_core/mapping.yaml b/tests/default/_core/mapping.yaml similarity index 95% rename from tests/_core/mapping.yaml rename to tests/default/_core/mapping.yaml index 936cf021e..29c2c335d 100644 --- a/tests/_core/mapping.yaml +++ b/tests/default/_core/mapping.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test mappings endpoints. prologues: diff --git a/tests/_core/reindex.yaml b/tests/default/_core/reindex.yaml similarity index 97% rename from tests/_core/reindex.yaml rename to tests/default/_core/reindex.yaml index 2b349d341..86d4fccb4 100644 --- a/tests/_core/reindex.yaml +++ b/tests/default/_core/reindex.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test reindex. epilogues: diff --git a/tests/_core/reindex/pipeline.yaml b/tests/default/_core/reindex/pipeline.yaml similarity index 97% rename from tests/_core/reindex/pipeline.yaml rename to tests/default/_core/reindex/pipeline.yaml index cdcae8230..bd63c249d 100644 --- a/tests/_core/reindex/pipeline.yaml +++ b/tests/default/_core/reindex/pipeline.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test reindex with a Search pipeline. epilogues: diff --git a/tests/_core/search/_source.yaml b/tests/default/_core/search/_source.yaml similarity index 98% rename from tests/_core/search/_source.yaml rename to tests/default/_core/search/_source.yaml index de34a143b..e0a4c7d04 100644 --- a/tests/_core/search/_source.yaml +++ b/tests/default/_core/search/_source.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint (_source). prologues: diff --git a/tests/_core/search/cancel_after_time_interval.yaml b/tests/default/_core/search/cancel_after_time_interval.yaml similarity index 90% rename from tests/_core/search/cancel_after_time_interval.yaml rename to tests/default/_core/search/cancel_after_time_interval.yaml index b5bd3b510..464c86a5a 100644 --- a/tests/_core/search/cancel_after_time_interval.yaml +++ b/tests/default/_core/search/cancel_after_time_interval.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with cancel_after_time_interval. prologues: diff --git a/tests/_core/search/geo_distance.yaml b/tests/default/_core/search/geo_distance.yaml similarity index 95% rename from tests/_core/search/geo_distance.yaml rename to tests/default/_core/search/geo_distance.yaml index 844df33c2..3beff6865 100644 --- a/tests/_core/search/geo_distance.yaml +++ b/tests/default/_core/search/geo_distance.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with geo_distance query. prologues: diff --git a/tests/_core/search/match.yaml b/tests/default/_core/search/match.yaml similarity index 95% rename from tests/_core/search/match.yaml rename to tests/default/_core/search/match.yaml index 2766f3ac1..978493787 100644 --- a/tests/_core/search/match.yaml +++ b/tests/default/_core/search/match.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with match query. prologues: diff --git a/tests/_core/search/multi_match.yaml b/tests/default/_core/search/multi_match.yaml similarity index 94% rename from tests/_core/search/multi_match.yaml rename to tests/default/_core/search/multi_match.yaml index b5c7b3c76..be83a9145 100644 --- a/tests/_core/search/multi_match.yaml +++ b/tests/default/_core/search/multi_match.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with multi_match query. prologues: diff --git a/tests/_core/search/phase_took.yaml b/tests/default/_core/search/phase_took.yaml similarity index 90% rename from tests/_core/search/phase_took.yaml rename to tests/default/_core/search/phase_took.yaml index 3e061ae0f..5da69c466 100644 --- a/tests/_core/search/phase_took.yaml +++ b/tests/default/_core/search/phase_took.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with phase_took. prologues: diff --git a/tests/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml similarity index 98% rename from tests/_core/search/rest_total_hits_as_int.yaml rename to tests/default/_core/search/rest_total_hits_as_int.yaml index 1885bc282..3d01d28da 100644 --- a/tests/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with rest_total_hits_as_int. prologues: diff --git a/tests/_core/search/seq_no_primary_term.yaml b/tests/default/_core/search/seq_no_primary_term.yaml similarity index 93% rename from tests/_core/search/seq_no_primary_term.yaml rename to tests/default/_core/search/seq_no_primary_term.yaml index 8082de549..8324f9850 100644 --- a/tests/_core/search/seq_no_primary_term.yaml +++ b/tests/default/_core/search/seq_no_primary_term.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with seq_no_primary_term. prologues: diff --git a/tests/_core/search/size.yaml b/tests/default/_core/search/size.yaml similarity index 95% rename from tests/_core/search/size.yaml rename to tests/default/_core/search/size.yaml index 64b2b8ca9..7bf68e2c2 100644 --- a/tests/_core/search/size.yaml +++ b/tests/default/_core/search/size.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test search endpoint with size. prologues: diff --git a/tests/cat/aliases.yaml b/tests/default/cat/aliases.yaml similarity index 93% rename from tests/cat/aliases.yaml rename to tests/default/cat/aliases.yaml index bef1d6bbc..30f9cebf9 100644 --- a/tests/cat/aliases.yaml +++ b/tests/default/cat/aliases.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/aliases endpoints. epilogues: diff --git a/tests/cat/allocation.yaml b/tests/default/cat/allocation.yaml similarity index 79% rename from tests/cat/allocation.yaml rename to tests/default/cat/allocation.yaml index e5d22d0a1..60d603d3b 100644 --- a/tests/cat/allocation.yaml +++ b/tests/default/cat/allocation.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/allocation endpoints. chapters: diff --git a/tests/cat/cluster_manager.yaml b/tests/default/cat/cluster_manager.yaml similarity index 81% rename from tests/cat/cluster_manager.yaml rename to tests/default/cat/cluster_manager.yaml index 3f0686d32..e984f2440 100644 --- a/tests/cat/cluster_manager.yaml +++ b/tests/default/cat/cluster_manager.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/cluster_manager endpoints. chapters: diff --git a/tests/cat/count.yaml b/tests/default/cat/count.yaml similarity index 78% rename from tests/cat/count.yaml rename to tests/default/cat/count.yaml index e5bc876ab..31e71c580 100644 --- a/tests/cat/count.yaml +++ b/tests/default/cat/count.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/count endpoints. chapters: diff --git a/tests/cat/fielddata.yaml b/tests/default/cat/fielddata.yaml similarity index 89% rename from tests/cat/fielddata.yaml rename to tests/default/cat/fielddata.yaml index 35a6721a1..ca8a11cb5 100644 --- a/tests/cat/fielddata.yaml +++ b/tests/default/cat/fielddata.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/fielddata endpoints. chapters: diff --git a/tests/cat/health.yaml b/tests/default/cat/health.yaml similarity index 98% rename from tests/cat/health.yaml rename to tests/default/cat/health.yaml index cc18865be..38865e9e5 100644 --- a/tests/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/health endpoints. chapters: diff --git a/tests/cat/index.yaml b/tests/default/cat/index.yaml similarity index 76% rename from tests/cat/index.yaml rename to tests/default/cat/index.yaml index 7127fe574..ae20cdf3a 100644 --- a/tests/cat/index.yaml +++ b/tests/default/cat/index.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat endpoints. chapters: diff --git a/tests/cat/indices.yaml b/tests/default/cat/indices.yaml similarity index 97% rename from tests/cat/indices.yaml rename to tests/default/cat/indices.yaml index c8d3fcb42..c20df6fe8 100644 --- a/tests/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/indices endpoints. prologues: diff --git a/tests/cat/master.yaml b/tests/default/cat/master.yaml similarity index 80% rename from tests/cat/master.yaml rename to tests/default/cat/master.yaml index b1ed7985b..435a27fa1 100644 --- a/tests/cat/master.yaml +++ b/tests/default/cat/master.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/master endpoints. chapters: diff --git a/tests/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml similarity index 84% rename from tests/cat/nodeattrs.yaml rename to tests/default/cat/nodeattrs.yaml index 98e1f8b29..cb9dc328c 100644 --- a/tests/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: diff --git a/tests/cat/nodes.yaml b/tests/default/cat/nodes.yaml similarity index 89% rename from tests/cat/nodes.yaml rename to tests/default/cat/nodes.yaml index 919100791..158354e23 100644 --- a/tests/cat/nodes.yaml +++ b/tests/default/cat/nodes.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodes endpoints. chapters: diff --git a/tests/cat/pending_tasks.yaml b/tests/default/cat/pending_tasks.yaml similarity index 79% rename from tests/cat/pending_tasks.yaml rename to tests/default/cat/pending_tasks.yaml index e76ad3329..c007487ad 100644 --- a/tests/cat/pending_tasks.yaml +++ b/tests/default/cat/pending_tasks.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/pending_tasks endpoints. chapters: diff --git a/tests/cat/pit_segments/all.yaml b/tests/default/cat/pit_segments/all.yaml similarity index 92% rename from tests/cat/pit_segments/all.yaml rename to tests/default/cat/pit_segments/all.yaml index 91b4ccdd1..5e6d77632 100644 --- a/tests/cat/pit_segments/all.yaml +++ b/tests/default/cat/pit_segments/all.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test cat/pit_segments/_all endpoints. version: '>= 2.4' diff --git a/tests/cat/pit_segments/pit_segments.yaml b/tests/default/cat/pit_segments/pit_segments.yaml similarity index 93% rename from tests/cat/pit_segments/pit_segments.yaml rename to tests/default/cat/pit_segments/pit_segments.yaml index 21ea56af0..d933ba55b 100644 --- a/tests/cat/pit_segments/pit_segments.yaml +++ b/tests/default/cat/pit_segments/pit_segments.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test cat/pit_segments endpoints. version: '>= 2.4' diff --git a/tests/cat/plugins.yaml b/tests/default/cat/plugins.yaml similarity index 82% rename from tests/cat/plugins.yaml rename to tests/default/cat/plugins.yaml index e2d665517..383576a7b 100644 --- a/tests/cat/plugins.yaml +++ b/tests/default/cat/plugins.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/plugins endpoints. chapters: diff --git a/tests/cat/recovery.yaml b/tests/default/cat/recovery.yaml similarity index 78% rename from tests/cat/recovery.yaml rename to tests/default/cat/recovery.yaml index ab4a9647b..a1341f219 100644 --- a/tests/cat/recovery.yaml +++ b/tests/default/cat/recovery.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/recovery endpoints. chapters: diff --git a/tests/cat/segment_replication.yaml b/tests/default/cat/segment_replication.yaml similarity index 81% rename from tests/cat/segment_replication.yaml rename to tests/default/cat/segment_replication.yaml index cdc2af500..0495ece5f 100644 --- a/tests/cat/segment_replication.yaml +++ b/tests/default/cat/segment_replication.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/segment_replication endpoints. version: '>= 2.7' diff --git a/tests/cat/segments.yaml b/tests/default/cat/segments.yaml similarity index 78% rename from tests/cat/segments.yaml rename to tests/default/cat/segments.yaml index 8d46443ec..b8cbb8f56 100644 --- a/tests/cat/segments.yaml +++ b/tests/default/cat/segments.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/segments endpoints. chapters: diff --git a/tests/cat/shards.yaml b/tests/default/cat/shards.yaml similarity index 90% rename from tests/cat/shards.yaml rename to tests/default/cat/shards.yaml index 9bf960b2e..f2ab5c147 100644 --- a/tests/cat/shards.yaml +++ b/tests/default/cat/shards.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/shards endpoints. epilogues: diff --git a/tests/cat/tasks.yaml b/tests/default/cat/tasks.yaml similarity index 78% rename from tests/cat/tasks.yaml rename to tests/default/cat/tasks.yaml index dbde807f1..aedcaab42 100644 --- a/tests/cat/tasks.yaml +++ b/tests/default/cat/tasks.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/tasks endpoints. chapters: diff --git a/tests/cat/templates.yaml b/tests/default/cat/templates.yaml similarity index 93% rename from tests/cat/templates.yaml rename to tests/default/cat/templates.yaml index 3f233a76b..330b6c12c 100644 --- a/tests/cat/templates.yaml +++ b/tests/default/cat/templates.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/templates endpoints. epilogues: diff --git a/tests/cat/thread_pool.yaml b/tests/default/cat/thread_pool.yaml similarity index 89% rename from tests/cat/thread_pool.yaml rename to tests/default/cat/thread_pool.yaml index 549286655..b38cd8772 100644 --- a/tests/cat/thread_pool.yaml +++ b/tests/default/cat/thread_pool.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/thread_pool endpoints. chapters: diff --git a/tests/cluster/component_template.yaml b/tests/default/cluster/component_template.yaml similarity index 97% rename from tests/cluster/component_template.yaml rename to tests/default/cluster/component_template.yaml index 783393591..0d8879d4f 100644 --- a/tests/cluster/component_template.yaml +++ b/tests/default/cluster/component_template.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test component templates. epilogues: diff --git a/tests/default/docker-compose.yml b/tests/default/docker-compose.yml new file mode 100644 index 000000000..04c267ec5 --- /dev/null +++ b/tests/default/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node diff --git a/tests/indices/alias/alias.yaml b/tests/default/indices/alias/alias.yaml similarity index 96% rename from tests/indices/alias/alias.yaml rename to tests/default/indices/alias/alias.yaml index 09962aff6..2fbba5383 100644 --- a/tests/indices/alias/alias.yaml +++ b/tests/default/indices/alias/alias.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test alias endpoints. epilogues: diff --git a/tests/indices/aliases/aliases.yaml b/tests/default/indices/aliases/aliases.yaml similarity index 93% rename from tests/indices/aliases/aliases.yaml rename to tests/default/indices/aliases/aliases.yaml index 22bcd95f1..805510ae4 100644 --- a/tests/indices/aliases/aliases.yaml +++ b/tests/default/indices/aliases/aliases.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test aliases endpoints. epilogues: diff --git a/tests/indices/aliases/put_alias.yaml b/tests/default/indices/aliases/put_alias.yaml similarity index 96% rename from tests/indices/aliases/put_alias.yaml rename to tests/default/indices/aliases/put_alias.yaml index 7a7099efa..d7eaa5c99 100644 --- a/tests/indices/aliases/put_alias.yaml +++ b/tests/default/indices/aliases/put_alias.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test put_alias variants. epilogues: diff --git a/tests/indices/cache.yaml b/tests/default/indices/cache.yaml similarity index 95% rename from tests/indices/cache.yaml rename to tests/default/indices/cache.yaml index d88e81497..db4db9dd5 100644 --- a/tests/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. prologues: diff --git a/tests/indices/clone.yaml b/tests/default/indices/clone.yaml similarity index 95% rename from tests/indices/clone.yaml rename to tests/default/indices/clone.yaml index dfc56119c..e2285c536 100644 --- a/tests/indices/clone.yaml +++ b/tests/default/indices/clone.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cloning an index. prologues: diff --git a/tests/indices/close.yaml b/tests/default/indices/close.yaml similarity index 95% rename from tests/indices/close.yaml rename to tests/default/indices/close.yaml index ca0a517d9..c8b9e8989 100644 --- a/tests/indices/close.yaml +++ b/tests/default/indices/close.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test closing an index. prologues: diff --git a/tests/indices/component_template.yaml b/tests/default/indices/component_template.yaml similarity index 98% rename from tests/indices/component_template.yaml rename to tests/default/indices/component_template.yaml index d05b88f4c..ff7b2d39f 100644 --- a/tests/indices/component_template.yaml +++ b/tests/default/indices/component_template.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test component templates. epilogues: diff --git a/tests/indices/dangling.yaml b/tests/default/indices/dangling.yaml similarity index 68% rename from tests/indices/dangling.yaml rename to tests/default/indices/dangling.yaml index 5accfcce8..94087d494 100644 --- a/tests/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. chapters: diff --git a/tests/indices/data_stream/data_stream.yaml b/tests/default/indices/data_stream/data_stream.yaml similarity index 95% rename from tests/indices/data_stream/data_stream.yaml rename to tests/default/indices/data_stream/data_stream.yaml index 051451286..b03845e21 100644 --- a/tests/indices/data_stream/data_stream.yaml +++ b/tests/default/indices/data_stream/data_stream.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test data streams. prologues: diff --git a/tests/indices/data_stream/rollover.yaml b/tests/default/indices/data_stream/rollover.yaml similarity index 93% rename from tests/indices/data_stream/rollover.yaml rename to tests/default/indices/data_stream/rollover.yaml index f8640db10..5e0c637be 100644 --- a/tests/indices/data_stream/rollover.yaml +++ b/tests/default/indices/data_stream/rollover.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test data streams rollover. prologues: diff --git a/tests/indices/data_stream/stats.yaml b/tests/default/indices/data_stream/stats.yaml similarity index 93% rename from tests/indices/data_stream/stats.yaml rename to tests/default/indices/data_stream/stats.yaml index 8052ff670..b8cbcd0bf 100644 --- a/tests/indices/data_stream/stats.yaml +++ b/tests/default/indices/data_stream/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test data streams. prologues: diff --git a/tests/indices/doc.yaml b/tests/default/indices/doc.yaml similarity index 95% rename from tests/indices/doc.yaml rename to tests/default/indices/doc.yaml index 325602d75..6eb5a8a5d 100644 --- a/tests/indices/doc.yaml +++ b/tests/default/indices/doc.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test inserting and retrieving a doc. epilogues: diff --git a/tests/indices/flush.yaml b/tests/default/indices/flush.yaml similarity index 94% rename from tests/indices/flush.yaml rename to tests/default/indices/flush.yaml index a38c4c65b..c2afb3af4 100644 --- a/tests/indices/flush.yaml +++ b/tests/default/indices/flush.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test flushing an index. prologues: diff --git a/tests/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml similarity index 96% rename from tests/indices/forcemerge.yaml rename to tests/default/indices/forcemerge.yaml index 0e884d1e5..d537e3a7c 100644 --- a/tests/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -1,5 +1,5 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. prologues: diff --git a/tests/indices/index.yaml b/tests/default/indices/index.yaml similarity index 97% rename from tests/indices/index.yaml rename to tests/default/indices/index.yaml index 186b7df8f..d30186c8a 100644 --- a/tests/indices/index.yaml +++ b/tests/default/indices/index.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test endpoints relevant the lifecycle of an index, from creation to deletion. epilogues: diff --git a/tests/indices/index_template.yaml b/tests/default/indices/index_template.yaml similarity index 97% rename from tests/indices/index_template.yaml rename to tests/default/indices/index_template.yaml index 09a6ce7f4..a3d5aa463 100644 --- a/tests/indices/index_template.yaml +++ b/tests/default/indices/index_template.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test index templates. epilogues: diff --git a/tests/indices/mapping.yml b/tests/default/indices/mapping.yml similarity index 98% rename from tests/indices/mapping.yml rename to tests/default/indices/mapping.yml index 00a11407d..ac340f93e 100644 --- a/tests/indices/mapping.yml +++ b/tests/default/indices/mapping.yml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test mappings endpoints. prologues: diff --git a/tests/indices/open.yaml b/tests/default/indices/open.yaml similarity index 96% rename from tests/indices/open.yaml rename to tests/default/indices/open.yaml index 61aa6fc4b..2253dddc1 100644 --- a/tests/indices/open.yaml +++ b/tests/default/indices/open.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test opening an index. prologues: diff --git a/tests/indices/refresh.yaml b/tests/default/indices/refresh.yaml similarity index 92% rename from tests/indices/refresh.yaml rename to tests/default/indices/refresh.yaml index 587168962..2303ed287 100644 --- a/tests/indices/refresh.yaml +++ b/tests/default/indices/refresh.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test _refresh. epilogues: diff --git a/tests/indices/settings.yaml b/tests/default/indices/settings.yaml similarity index 97% rename from tests/indices/settings.yaml rename to tests/default/indices/settings.yaml index a5835fdb0..5bfada4b9 100644 --- a/tests/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -1,5 +1,5 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test settings for an index. prologues: diff --git a/tests/indices/shrink.yaml b/tests/default/indices/shrink.yaml similarity index 97% rename from tests/indices/shrink.yaml rename to tests/default/indices/shrink.yaml index 1803889ab..ce35ae605 100644 --- a/tests/indices/shrink.yaml +++ b/tests/default/indices/shrink.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test shrinking an index. prologues: diff --git a/tests/indices/split.yaml b/tests/default/indices/split.yaml similarity index 97% rename from tests/indices/split.yaml rename to tests/default/indices/split.yaml index 980feb287..6590cf3e9 100644 --- a/tests/indices/split.yaml +++ b/tests/default/indices/split.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test splitting an index. prologues: diff --git a/tests/indices/stats.yaml b/tests/default/indices/stats.yaml similarity index 95% rename from tests/indices/stats.yaml rename to tests/default/indices/stats.yaml index 2874cae76..fa340ab79 100644 --- a/tests/indices/stats.yaml +++ b/tests/default/indices/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test stats for an index. prologues: diff --git a/tests/indices/update_by_query.yaml b/tests/default/indices/update_by_query.yaml similarity index 96% rename from tests/indices/update_by_query.yaml rename to tests/default/indices/update_by_query.yaml index a58853f3f..0ce1295d4 100644 --- a/tests/indices/update_by_query.yaml +++ b/tests/default/indices/update_by_query.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test reindex. epilogues: diff --git a/tests/ingest/pipeline.yaml b/tests/default/ingest/pipeline.yaml similarity index 94% rename from tests/ingest/pipeline.yaml rename to tests/default/ingest/pipeline.yaml index 5fd90d3ca..3145dcb62 100644 --- a/tests/ingest/pipeline.yaml +++ b/tests/default/ingest/pipeline.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of an ingest pipeline with a text embedding processor. epilogues: diff --git a/tests/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml similarity index 95% rename from tests/ml/model_groups.yaml rename to tests/default/ml/model_groups.yaml index 0e5e68b18..a08efa0ce 100644 --- a/tests/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. version: '>= 2.11' diff --git a/tests/ml/models.yaml b/tests/default/ml/models.yaml similarity index 94% rename from tests/ml/models.yaml rename to tests/default/ml/models.yaml index ecf8c3561..ac386c0a4 100644 --- a/tests/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. version: '>= 2.11' diff --git a/tests/nodes/info.yaml b/tests/default/nodes/info.yaml similarity index 84% rename from tests/nodes/info.yaml rename to tests/default/nodes/info.yaml index 362a04803..7e352f8a6 100644 --- a/tests/nodes/info.yaml +++ b/tests/default/nodes/info.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test nodes endpoints. chapters: diff --git a/tests/nodes/stats.yaml b/tests/default/nodes/stats.yaml similarity index 98% rename from tests/nodes/stats.yaml rename to tests/default/nodes/stats.yaml index ecb6152ea..a52225be7 100644 --- a/tests/nodes/stats.yaml +++ b/tests/default/nodes/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test nodes stats. chapters: diff --git a/tests/ppl/explain.yaml b/tests/default/ppl/explain.yaml similarity index 88% rename from tests/ppl/explain.yaml rename to tests/default/ppl/explain.yaml index 70df71d79..339123494 100644 --- a/tests/ppl/explain.yaml +++ b/tests/default/ppl/explain.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test how a query is executed against OpenSearch. diff --git a/tests/ppl/query.yaml b/tests/default/ppl/query.yaml similarity index 88% rename from tests/ppl/query.yaml rename to tests/default/ppl/query.yaml index a525a85a0..5e8bfe506 100644 --- a/tests/ppl/query.yaml +++ b/tests/default/ppl/query.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test send PPL query to index. diff --git a/tests/ppl/stats.yaml b/tests/default/ppl/stats.yaml similarity index 78% rename from tests/ppl/stats.yaml rename to tests/default/ppl/stats.yaml index ec2786432..fafba95ec 100644 --- a/tests/ppl/stats.yaml +++ b/tests/default/ppl/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test get SQL Queries stats. diff --git a/tests/sql/close.yaml b/tests/default/sql/close.yaml similarity index 95% rename from tests/sql/close.yaml rename to tests/default/sql/close.yaml index b82b50be6..bb8e8d06f 100644 --- a/tests/sql/close.yaml +++ b/tests/default/sql/close.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test to explicitly clear the cursor context. diff --git a/tests/sql/explain.yaml b/tests/default/sql/explain.yaml similarity index 88% rename from tests/sql/explain.yaml rename to tests/default/sql/explain.yaml index 22a648f58..3ac74d222 100644 --- a/tests/sql/explain.yaml +++ b/tests/default/sql/explain.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test how a query is executed against OpenSearch. diff --git a/tests/sql/query.yaml b/tests/default/sql/query.yaml similarity index 89% rename from tests/sql/query.yaml rename to tests/default/sql/query.yaml index 96bf4ee43..fdc5e2c85 100644 --- a/tests/sql/query.yaml +++ b/tests/default/sql/query.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test send SQL query to index. diff --git a/tests/sql/settings.yaml b/tests/default/sql/settings.yaml similarity index 94% rename from tests/sql/settings.yaml rename to tests/default/sql/settings.yaml index 6a826806f..4de2b1158 100644 --- a/tests/sql/settings.yaml +++ b/tests/default/sql/settings.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test updating SQL settings in OS cluster. diff --git a/tests/sql/stats.yaml b/tests/default/sql/stats.yaml similarity index 78% rename from tests/sql/stats.yaml rename to tests/default/sql/stats.yaml index cbb8d7db1..87a181b44 100644 --- a/tests/sql/stats.yaml +++ b/tests/default/sql/stats.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test get SQL Queries stats. diff --git a/.github/opensearch-cluster/docker-compose.yml b/tests/plugins/index_state_management/docker-compose.yml similarity index 92% rename from .github/opensearch-cluster/docker-compose.yml rename to tests/plugins/index_state_management/docker-compose.yml index e2ea4b2de..a9db789bc 100644 --- a/.github/opensearch-cluster/docker-compose.yml +++ b/tests/plugins/index_state_management/docker-compose.yml @@ -10,5 +10,4 @@ services: - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} - discovery.type=single-node - - path.repo=/tmp/opensearch/repo - plugins.index_state_management.job_interval=1 \ No newline at end of file diff --git a/tests/nodes/plugins/index_state_management.yaml b/tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml similarity index 90% rename from tests/nodes/plugins/index_state_management.yaml rename to tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml index 4f95866d6..7e22dea77 100644 --- a/tests/nodes/plugins/index_state_management.yaml +++ b/tests/plugins/index_state_management/nodes/plugins/index_state_management.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../json_schemas/test_story.schema.yaml description: Get index_state_management node info settings. prologues: diff --git a/tests/cat/repositories.yaml b/tests/snapshot/cat/repositories.yaml similarity index 92% rename from tests/cat/repositories.yaml rename to tests/snapshot/cat/repositories.yaml index 72cd80dce..89d258e23 100644 --- a/tests/cat/repositories.yaml +++ b/tests/snapshot/cat/repositories.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/repositories endpoints. epilogues: diff --git a/tests/cat/snapshots.yaml b/tests/snapshot/cat/snapshots.yaml similarity index 90% rename from tests/cat/snapshots.yaml rename to tests/snapshot/cat/snapshots.yaml index 962827112..1e35ea574 100644 --- a/tests/cat/snapshots.yaml +++ b/tests/snapshot/cat/snapshots.yaml @@ -1,4 +1,4 @@ -$schema: ../../json_schemas/test_story.schema.yaml +$schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/snapshots endpoints. epilogues: diff --git a/tests/snapshot/docker-compose.yml b/tests/snapshot/docker-compose.yml new file mode 100644 index 000000000..86cb0b9b3 --- /dev/null +++ b/tests/snapshot/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node + - path.repo=/tmp/opensearch/repo diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 8962931f4..fdde53791 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -18,12 +18,18 @@ import { basename, resolve } from 'path' import type StoryValidator from "./StoryValidator"; import { OpenSearchHttpClient } from 'OpenSearchHttpClient' import * as ansi from './Ansi' +import _ from 'lodash' + +const EXCLUDED_FILES = [ + 'docker-compose.yml' +] export default class TestRunner { private readonly _http_client: OpenSearchHttpClient private readonly _story_validator: StoryValidator private readonly _story_evaluator: StoryEvaluator private readonly _result_logger: ResultLogger + private readonly _story_files: Record = {} constructor (http_client: OpenSearchHttpClient, story_validator: StoryValidator, story_evaluator: StoryEvaluator, result_logger: ResultLogger) { this._http_client = http_client @@ -34,7 +40,7 @@ export default class TestRunner { async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false - const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', '')) + const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { @@ -53,6 +59,12 @@ export default class TestRunner { return { results, failed } } + story_files(story_path: string): StoryFile[] { + if (this._story_files[story_path] !== undefined) return this._story_files[story_path] + this._story_files[story_path] = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', '')) + return this._story_files[story_path] + } + #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` @@ -64,9 +76,11 @@ export default class TestRunner { story }] } else { - return fs.readdirSync(path).flatMap(next_file => { - return this.#collect_story_files(path, next_file, next_prefix) - }) + return _.compact(fs.readdirSync(path).flatMap(next_file => { + if (!EXCLUDED_FILES.includes(next_file)) { + return this.#collect_story_files(path, next_file, next_prefix) + } + })) } } diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 601d3ff5d..a05800b37 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -33,7 +33,7 @@ import TestResults from './TestResults' const command = new Command() .description('Run test stories against the OpenSearch spec.') .addOption(new Option('--spec, --spec-path ', 'path to the root folder of the multi-file spec').default('./spec')) - .addOption(new Option('--tests, --tests-path ', 'path to the root folder of the tests').default('./tests')) + .addOption(new Option('--tests, --tests-path ', 'path to the root folder of the tests').default('./tests/default')) .addOption( new Option('--tab-width ', 'tab width for displayed results') .default(4) diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 99d1e1ba3..b0333504f 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -39,3 +39,13 @@ test('stories folder', async () => { const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped] expect(actual_evaluations).toEqual(expected_evaluations) }) + +describe('story_files', () => { + const { test_runner } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') + + test('does not contain docker-compose.yml', () => { + expect(test_runner.story_files('tests/plugins/index_state_management').map( + story_file => story_file.display_path + )).not.toContain('nodes/plugins/docker-compose.yml') + }) +}) From e865cf6755dc551e96fbd2f4252fc50a869db01a Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:42:43 -0500 Subject: [PATCH 11/31] Add Segments API test (#477) * Add Segments API test Signed-off-by: Archer * Fix errors Signed-off-by: Archer * Update changelog Signed-off-by: Archer --------- Signed-off-by: Archer --- tests/default/indices/segments.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/default/indices/segments.yaml diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml new file mode 100644 index 000000000..da87fec7b --- /dev/null +++ b/tests/default/indices/segments.yaml @@ -0,0 +1,28 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: This story tests the Segments API. + +prologues: + - path: /movies + method: PUT +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get details about Lucene segments. + path: /_segments + method: GET + parameters: + expand_wildcards: none + response: + status: 200 + - synopsis: Get details about Lucene segments inside the specified index. + path: /{index}/_segments + method: GET + parameters: + index: movies + expand_wildcards: all + allow_no_indices: false + response: + status: 200 From 18587f56d95de28f0229a4fe98dfd6d15b850f62 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 12:59:06 -0400 Subject: [PATCH 12/31] Added support for SigV4. (#476) * Added support for running tests against Amazon OpenSearch. Signed-off-by: dblock --- .cspell | 5 +- CHANGELOG.md | 1 + TESTING_GUIDE.md | 30 +- eslint.config.mjs | 6 +- package-lock.json | 2251 ++++++++++++++++- package.json | 2 + tests/default/cat/plugins.yaml | 2 - tools/src/OpenSearchHttpClient.ts | 116 +- .../dump-cluster-spec/dump-cluster-spec.ts | 2 +- tools/src/tester/test.ts | 12 +- .../tests/tester/OpenSearchHttpClient.test.ts | 66 + tools/tests/tester/helpers.ts | 6 +- 12 files changed, 2392 insertions(+), 107 deletions(-) create mode 100644 tools/tests/tester/OpenSearchHttpClient.test.ts diff --git a/.cspell b/.cspell index 080c3937b..1542876a8 100644 --- a/.cspell +++ b/.cspell @@ -2,6 +2,7 @@ aarch actiongroup actiongroups aggregatable +aoss APIV argjson asciifolding @@ -129,6 +130,7 @@ readingform rebalance Rebalance recoverysource +Refn reindex Reindex relo @@ -183,5 +185,4 @@ urldecode vectory whoamiprotected wordnet -Yrtsd -Refn +Yrtsd \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e0e18a2..f5590437d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) +- Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) ### Changed diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index d50d4f5b8..5c066549e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,9 +1,11 @@ - [Spec Testing Guide](#spec-testing-guide) - - [Running Spec Tests Locally](#running-spec-tests-locally) - - [Common Errors](#common-errors) - - [401 Unauthorized](#401-unauthorized) - - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + - [Running Spec Tests](#running-spec-tests) + - [Running Spec Tests Locally](#running-spec-tests-locally) + - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) + - [Common Errors](#common-errors) + - [401 Unauthorized](#401-unauthorized) + - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) - [Writing Spec Tests](#writing-spec-tests) - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) @@ -18,7 +20,9 @@ We have devised our own test framework to test the spec against an OpenSearch cluster. We're still adding more features to the framework as the needs arise, and this document will be updated accordingly. This test framework has also been integrated into the repo's CI/CD pipeline. Checkout the [test-spec](.github/workflows/test-spec.yml) workflow for more details. -## Running Spec Tests Locally +## Running Spec Tests + +### Running Spec Tests Locally Set up an OpenSearch cluster with Docker: @@ -44,11 +48,25 @@ Verbose output: npm run test:spec -- --opensearch-insecure --verbose ``` +### Running Spec Tests with Amazon OpenSearch + +Use an Amazon OpenSearch service instance. + +```bash +export AWS_ACCESS_KEY_ID=<> +export AWS_SECRET_ACCESS_KEY=<> +export AWS_SESSION_TOKEN=<> +export AWS_REGION=us-west-2 +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec +``` + ### Common Errors #### 401 Unauthorized -Remember to set the `OPENSEARCH_PASSWORD` environment variable everytime you start a new shell to run the tests. +Remember to set the `OPENSEARCH_PASSWORD` or `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables every time you start a new shell to run the tests. #### FORBIDDEN/10/cluster create-index blocked (api) diff --git a/eslint.config.mjs b/eslint.config.mjs index 337f14721..f782d0a2b 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -8,6 +8,7 @@ import pluginTs from '@typescript-eslint/eslint-plugin' import pluginYml from 'eslint-plugin-yml' import pluginCspell from '@cspell/eslint-plugin' import pluginStylistic from '@stylistic/eslint-plugin' +import pluginJest from 'eslint-plugin-jest' export default [ pluginJs.configs.recommended, @@ -28,7 +29,8 @@ export default [ 'license-header': pluginLicenseHeader, 'eslint-comments': pluginComments, '@cspell': pluginCspell, - '@stylistic': pluginStylistic + '@stylistic': pluginStylistic, + 'jest': pluginJest }, rules: { ...pluginJs.configs.recommended.rules, @@ -78,6 +80,8 @@ export default [ ], '@typescript-eslint/require-await': 'error', '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + '@typescript-eslint/unbound-method': 'off', + 'jest/unbound-method': 'error', 'array-callback-return': 'off', 'indent': ['error', 2, { 'SwitchCase': 1 }], 'new-cap': 'off', diff --git a/package-lock.json b/package-lock.json index f86e80d30..e1156510e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", + "aws4-axios": "^3.3.7", "axios": "^1.7.1", "cbor": "^9.0.2", "commander": "^12.0.0", @@ -33,6 +34,7 @@ "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", "eslint-plugin-import": "^2.29.1", + "eslint-plugin-jest": "^28.8.0", "eslint-plugin-license-header": "^0.6.1", "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", @@ -104,6 +106,578 @@ "openapi-types": ">=7" } }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.624.0.tgz", + "integrity": "sha512-EX6EF+rJzMPC5dcdsu40xSi2To7GSvdGQNIpe97pD9WvZwM9tRNQnNM4T6HA4gjV1L6Jwk8rBlG/CnveXtLEMw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.624.0.tgz", + "integrity": "sha512-Ki2uKYJKKtfHxxZsiMTOvJoVRP6b2pZ1u3rcUb2m/nVgBPUfLdl8ZkGpqE29I+t5/QaS/sEdbn6cgMUZwl+3Dg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.624.0" + } + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.624.0.tgz", + "integrity": "sha512-k36fLZCb2nfoV/DKK3jbRgO/Yf7/R80pgYfMiotkGjnZwDmRvNN08z4l06L9C+CieazzkgRxNUzyppsYcYsQaw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.624.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.624.0.tgz", + "integrity": "sha512-WyFmPbhRIvtWi7hBp8uSFy+iPpj8ccNV/eX86hwF4irMjfc/FtsGVIAeBXxXM/vGCjkdfEzOnl+tJ2XACD4OXg==", + "dependencies": { + "@smithy/core": "^2.3.2", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.622.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.622.0.tgz", + "integrity": "sha512-VUHbr24Oll1RK3WR8XLUugLpgK9ZuxEm/NVeVqyFts1Ck9gsKpRg1x4eH7L7tW3SJ4TDEQNMbD7/7J+eoL2svg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.624.0.tgz", + "integrity": "sha512-mMoNIy7MO2WTBbdqMyLpbt6SZpthE6e0GkRYpsd0yozPt0RZopcBhEh+HG1U9Y1PVODo+jcMk353vAi61CfnhQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.624.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.624.0.tgz", + "integrity": "sha512-vYyGK7oNpd81BdbH5IlmQ6zfaQqU+rPwsKTDDBeLRjshtrGXOEpfoahVpG9PX0ibu32IOWp4ZyXBNyVrnvcMOw==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-ini": "3.624.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.624.0.tgz", + "integrity": "sha512-A02bayIjU9APEPKr3HudrFHEx0WfghoSPsPopckDkW7VBqO4wizzcxr75Q9A3vNX+cwg0wCN6UitTNe6pVlRaQ==", + "dependencies": { + "@aws-sdk/client-sso": "3.624.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.621.0" + } + }, + "node_modules/@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.620.0.tgz", + "integrity": "sha512-bvS6etn+KsuL32ubY5D3xNof1qkenpbJXf/ugGXbg0n98DvDFQ/F+SMLxHgbnER5dsKYchNnhmtI6/FC3HFu/A==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.614.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.6.tgz", @@ -1767,50 +2341,562 @@ "fastq": "^1.6.0" }, "engines": { - "node": ">= 8" + "node": ">= 8" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "peer": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "peer": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.3.2.tgz", + "integrity": "sha512-in5wwt6chDBcUv1Lw1+QzZxN9fBffi+qOixfb65yK4sDuKG7zAUO9HAFqmVzsZM3N+3tTyvZjtnDXePpvp007Q==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "dependencies": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-retry": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.14.tgz", + "integrity": "sha512-7ZaWZJOjUxa5hgmuMspyt8v/zVsh0GXYuF7OvCmdcbVa/xbnKQoYC+uYKunAqRGTkxjOyuOCw9rmFUFOqqC0eQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "dependencies": { + "@smithy/types": "^3.3.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/smithy-client": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.12.tgz", + "integrity": "sha512-wtm8JtsycthkHy1YA4zjIh2thJgIQ9vGkoR639DBx5lLlLNU0v4GARpQZkr2WjXue74nZ7MiTSWfVrLkyD8RkA==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.14.tgz", + "integrity": "sha512-0iwTgKKmAIf+vFLV8fji21Jb2px11ktKVxbX6LIDPAUJyWQqGqBVfwba7xwa1f2FZUoolYQgLvxQEpJycXuQ5w==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.14.tgz", + "integrity": "sha512-e9uQarJKfXApkTMMruIdxHprhcXivH1flYCe8JRDTzkkLx8dA3V5J8GZlST9yfDiRWkJpZJlUXGN9Rc9Ade3OQ==", + "dependencies": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, + "node_modules/@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, "engines": { - "node": ">=14" + "node": ">=16.0.0" } }, - "node_modules/@pkgr/core": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", - "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + "node_modules/@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "dependencies": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" }, - "funding": { - "url": "https://opencollective.com/unts" + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "peer": true, + "node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", "dependencies": { - "type-detect": "4.0.8" + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "peer": true, + "node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", "dependencies": { - "@sinonjs/commons": "^3.0.0" + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@stylistic/eslint-plugin": { @@ -2819,6 +3905,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws4": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz", + "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==" + }, + "node_modules/aws4-axios": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/aws4-axios/-/aws4-axios-3.3.7.tgz", + "integrity": "sha512-TVuUKe6SNv5vqrE6VLSR+eN8JJhijA28qKdMGKbY23TcksYl0KGrW9dADWRwLUiLTrKq/rTucjaoJzNruNf4RQ==", + "dependencies": { + "@aws-sdk/client-sts": "^3.4.1", + "aws4": "^1.12.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "axios": ">=1.6.0" + } + }, "node_modules/axios": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", @@ -2955,6 +4061,11 @@ "resolved": "https://registry.npmjs.org/bit-buffer/-/bit-buffer-0.2.5.tgz", "integrity": "sha512-x1yGnmXvFg6e3DiyRztElbcn1bsCTFSoM/ncAzY62uE0JdTl5xlKJd0ooqLYoPbhdsnpehSIQrdIvclcZJYwiA==" }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4170,6 +5281,30 @@ "semver": "bin/semver.js" } }, + "node_modules/eslint-plugin-jest": { + "version": "28.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-28.8.0.tgz", + "integrity": "sha512-Tubj1hooFxCl52G4qQu0edzV/+EZzPUeN8p2NnW5uu4fbDs+Yo7+qDVDc4/oG3FbCqEBmu/OC3LSsyiU22oghw==", + "dependencies": { + "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "@typescript-eslint/eslint-plugin": "^6.0.0 || ^7.0.0 || ^8.0.0", + "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0", + "jest": "*" + }, + "peerDependenciesMeta": { + "@typescript-eslint/eslint-plugin": { + "optional": true + }, + "jest": { + "optional": true + } + } + }, "node_modules/eslint-plugin-license-header": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/eslint-plugin-license-header/-/eslint-plugin-license-header-0.6.1.tgz", @@ -4602,6 +5737,27 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, + "node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -7611,6 +8767,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -8049,6 +9210,18 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -8289,61 +9462,533 @@ "node": ">=6" } }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + }, + "dependencies": { + "@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "peer": true, + "requires": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "@apidevtools/json-schema-ref-parser": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", + "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "requires": { + "@jsdevtools/ono": "^7.1.3", + "call-me-maybe": "^1.0.1", + "js-yaml": "^3.13.1" + } + }, + "@apidevtools/openapi-schemas": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", + "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==" + }, + "@apidevtools/swagger-methods": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", + "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + }, + "@apidevtools/swagger-parser": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz", + "integrity": "sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==", + "requires": { + "@apidevtools/json-schema-ref-parser": "9.0.6", + "@apidevtools/openapi-schemas": "^2.1.0", + "@apidevtools/swagger-methods": "^3.0.2", + "@jsdevtools/ono": "^7.1.3", + "ajv": "^8.6.3", + "ajv-draft-04": "^1.0.0", + "call-me-maybe": "^1.0.1" + } + }, + "@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "requires": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + } + } + }, + "@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "requires": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + } + }, + "@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "requires": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + } + } + }, + "@aws-sdk/client-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.624.0.tgz", + "integrity": "sha512-EX6EF+rJzMPC5dcdsu40xSi2To7GSvdGQNIpe97pD9WvZwM9tRNQnNM4T6HA4gjV1L6Jwk8rBlG/CnveXtLEMw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/client-sso-oidc": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.624.0.tgz", + "integrity": "sha512-Ki2uKYJKKtfHxxZsiMTOvJoVRP6b2pZ1u3rcUb2m/nVgBPUfLdl8ZkGpqE29I+t5/QaS/sEdbn6cgMUZwl+3Dg==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/client-sts": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.624.0.tgz", + "integrity": "sha512-k36fLZCb2nfoV/DKK3jbRgO/Yf7/R80pgYfMiotkGjnZwDmRvNN08z4l06L9C+CieazzkgRxNUzyppsYcYsQaw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.624.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/core": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.624.0.tgz", + "integrity": "sha512-WyFmPbhRIvtWi7hBp8uSFy+iPpj8ccNV/eX86hwF4irMjfc/FtsGVIAeBXxXM/vGCjkdfEzOnl+tJ2XACD4OXg==", + "requires": { + "@smithy/core": "^2.3.2", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-http": { + "version": "3.622.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.622.0.tgz", + "integrity": "sha512-VUHbr24Oll1RK3WR8XLUugLpgK9ZuxEm/NVeVqyFts1Ck9gsKpRg1x4eH7L7tW3SJ4TDEQNMbD7/7J+eoL2svg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-ini": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.624.0.tgz", + "integrity": "sha512-mMoNIy7MO2WTBbdqMyLpbt6SZpthE6e0GkRYpsd0yozPt0RZopcBhEh+HG1U9Y1PVODo+jcMk353vAi61CfnhQ==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-node": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.624.0.tgz", + "integrity": "sha512-vYyGK7oNpd81BdbH5IlmQ6zfaQqU+rPwsKTDDBeLRjshtrGXOEpfoahVpG9PX0ibu32IOWp4ZyXBNyVrnvcMOw==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-ini": "3.624.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.624.0.tgz", + "integrity": "sha512-A02bayIjU9APEPKr3HudrFHEx0WfghoSPsPopckDkW7VBqO4wizzcxr75Q9A3vNX+cwg0wCN6UitTNe6pVlRaQ==", + "requires": { + "@aws-sdk/client-sso": "3.624.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-user-agent": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.620.0.tgz", + "integrity": "sha512-bvS6etn+KsuL32ubY5D3xNof1qkenpbJXf/ugGXbg0n98DvDFQ/F+SMLxHgbnER5dsKYchNnhmtI6/FC3HFu/A==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } - } - }, - "dependencies": { - "@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "peer": true, + }, + "@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", "requires": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } }, - "@apidevtools/json-schema-ref-parser": { - "version": "9.0.6", - "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", - "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", "requires": { - "@jsdevtools/ono": "^7.1.3", - "call-me-maybe": "^1.0.1", - "js-yaml": "^3.13.1" + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" } }, - "@apidevtools/openapi-schemas": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", - "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==" + "@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "requires": { + "tslib": "^2.6.2" + } }, - "@apidevtools/swagger-methods": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", - "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + "@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } }, - "@apidevtools/swagger-parser": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz", - "integrity": "sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==", + "@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", "requires": { - "@apidevtools/json-schema-ref-parser": "9.0.6", - "@apidevtools/openapi-schemas": "^2.1.0", - "@apidevtools/swagger-methods": "^3.0.2", - "@jsdevtools/ono": "^7.1.3", - "ajv": "^8.6.3", - "ajv-draft-04": "^1.0.0", - "call-me-maybe": "^1.0.1" + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } }, "@babel/code-frame": { @@ -9702,6 +11347,413 @@ "@sinonjs/commons": "^3.0.0" } }, + "@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/core": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.3.2.tgz", + "integrity": "sha512-in5wwt6chDBcUv1Lw1+QzZxN9fBffi+qOixfb65yK4sDuKG7zAUO9HAFqmVzsZM3N+3tTyvZjtnDXePpvp007Q==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "requires": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-retry": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.14.tgz", + "integrity": "sha512-7ZaWZJOjUxa5hgmuMspyt8v/zVsh0GXYuF7OvCmdcbVa/xbnKQoYC+uYKunAqRGTkxjOyuOCw9rmFUFOqqC0eQ==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "requires": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "requires": { + "@smithy/types": "^3.3.0" + } + }, + "@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/smithy-client": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.12.tgz", + "integrity": "sha512-wtm8JtsycthkHy1YA4zjIh2thJgIQ9vGkoR639DBx5lLlLNU0v4GARpQZkr2WjXue74nZ7MiTSWfVrLkyD8RkA==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + } + }, + "@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "requires": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-browser": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.14.tgz", + "integrity": "sha512-0iwTgKKmAIf+vFLV8fji21Jb2px11ktKVxbX6LIDPAUJyWQqGqBVfwba7xwa1f2FZUoolYQgLvxQEpJycXuQ5w==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-node": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.14.tgz", + "integrity": "sha512-e9uQarJKfXApkTMMruIdxHprhcXivH1flYCe8JRDTzkkLx8dA3V5J8GZlST9yfDiRWkJpZJlUXGN9Rc9Ade3OQ==", + "requires": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "requires": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "requires": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + } + }, "@stylistic/eslint-plugin": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.3.0.tgz", @@ -10391,6 +12443,20 @@ "possible-typed-array-names": "^1.0.0" } }, + "aws4": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz", + "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==" + }, + "aws4-axios": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/aws4-axios/-/aws4-axios-3.3.7.tgz", + "integrity": "sha512-TVuUKe6SNv5vqrE6VLSR+eN8JJhijA28qKdMGKbY23TcksYl0KGrW9dADWRwLUiLTrKq/rTucjaoJzNruNf4RQ==", + "requires": { + "@aws-sdk/client-sts": "^3.4.1", + "aws4": "^1.12.0" + } + }, "axios": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", @@ -10502,6 +12568,11 @@ "resolved": "https://registry.npmjs.org/bit-buffer/-/bit-buffer-0.2.5.tgz", "integrity": "sha512-x1yGnmXvFg6e3DiyRztElbcn1bsCTFSoM/ncAzY62uE0JdTl5xlKJd0ooqLYoPbhdsnpehSIQrdIvclcZJYwiA==" }, + "bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -11444,6 +13515,14 @@ } } }, + "eslint-plugin-jest": { + "version": "28.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-28.8.0.tgz", + "integrity": "sha512-Tubj1hooFxCl52G4qQu0edzV/+EZzPUeN8p2NnW5uu4fbDs+Yo7+qDVDc4/oG3FbCqEBmu/OC3LSsyiU22oghw==", + "requires": { + "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, "eslint-plugin-license-header": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/eslint-plugin-license-header/-/eslint-plugin-license-header-0.6.1.tgz", @@ -11670,6 +13749,14 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, + "fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -13755,6 +15842,11 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -14032,6 +16124,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index d73f0868d..22d798e13 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", + "aws4-axios": "^3.3.7", "axios": "^1.7.1", "cbor": "^9.0.2", "commander": "^12.0.0", @@ -44,6 +45,7 @@ "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", "eslint-plugin-import": "^2.29.1", + "eslint-plugin-jest": "^28.8.0", "eslint-plugin-license-header": "^0.6.1", "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", diff --git a/tests/default/cat/plugins.yaml b/tests/default/cat/plugins.yaml index 383576a7b..a2fb2b42f 100644 --- a/tests/default/cat/plugins.yaml +++ b/tests/default/cat/plugins.yaml @@ -9,5 +9,3 @@ chapters: format: json response: status: 200 - payload: - - component: opensearch-alerting diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 2652cd101..f81656074 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -11,6 +11,8 @@ import { Option } from '@commander-js/extra-typings' import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios' import * as https from 'node:https' import { sleep } from './helpers' +import { Logger } from './Logger' +import { aws4Interceptor } from 'aws4-axios' const DEFAULT_URL = 'https://localhost:9200' const DEFAULT_USER = 'admin' @@ -30,23 +32,76 @@ export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password ', 'AWS access key ID') + .env('AWS_ACCESS_KEY_ID') + +export const AWS_SECRET_ACCESS_KEY_OPTION = new Option('--aws-secret-access-key ', 'AWS secret access key') + .env('AWS_SECRET_ACCESS_KEY') + +export const AWS_SESSION_TOKEN_OPTION = new Option('--aws-session-token ', 'AWS session token') + .env('AWS_SESSION_TOKEN') + +export const AWS_REGION_OPTION = new Option('--aws-region ', 'AWS region') + .env('AWS_REGION') + .default('us-east-1') + +export const AWS_SERVICE_OPTION = new Option('--aws-service ', 'AWS service ID') + .env('AWS_SERVICE') + .default('es') + +export interface BasicAuth { + username: string + password: string +} + +export interface AwsAuth { + aws_access_key_id: string + aws_access_secret_key: string + aws_access_session_token?: string + aws_region?: string + aws_service?: string +} + export interface OpenSearchHttpClientOptions { url?: string - username?: string - password?: string - insecure?: boolean, - responseType: ResponseType | undefined + insecure?: boolean + responseType?: ResponseType + logger?: Logger, + basic_auth?: BasicAuth + aws_auth?: AwsAuth } -export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize}`]: OpenSearchHttpClientOptions[K] } +export type OpenSearchHttpClientCliOptions = { + opensearchUrl?: string + opensearchUsername?: string + opensearchPassword?: string + opensearchInsecure?: boolean + awsAccessKeyId?: string + awsSecretAccessKey?: string + awsSessionToken?: string + awsRegion?: string + awsService?: string + responseType?: ResponseType + logger?: Logger +} export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptions): OpenSearchHttpClientOptions { return { url: opts.opensearchUrl, - username: opts.opensearchUsername, - password: opts.opensearchPassword, insecure: opts.opensearchInsecure, - responseType: opts.opensearchResponseType + basic_auth: opts.opensearchUsername !== undefined && opts.opensearchPassword !== undefined ? { + username: opts.opensearchUsername, + password: opts.opensearchPassword + } : undefined, + aws_auth: opts.awsAccessKeyId !== undefined && opts.awsSecretAccessKey !== undefined ? { + aws_access_key_id: opts?.awsAccessKeyId, + aws_access_secret_key: opts?.awsSecretAccessKey, + aws_access_session_token: opts?.awsSessionToken, + aws_region: opts?.awsRegion, + aws_service: opts?.awsService, + } : undefined, + responseType: opts.responseType, + logger: opts?.logger } } @@ -72,26 +127,52 @@ export interface OpenSearchInfo { export class OpenSearchHttpClient { private readonly _axios: AxiosInstance private readonly _opts?: OpenSearchHttpClientOptions + private readonly _logger: Logger constructor (opts?: OpenSearchHttpClientOptions) { this._opts = opts + this._logger = opts?.logger ?? new Logger() + + let auth = undefined + let sigv4_interceptor = undefined + + if (opts?.basic_auth !== undefined) { + this._logger.info(`Authenticating with ${opts.basic_auth.username} ...`) + auth = opts.basic_auth + } else if (opts?.aws_auth !== undefined) { + this._logger.info(`Authenticating using SigV4 with ${opts.aws_auth.aws_access_key_id} (${opts.aws_auth.aws_region}) ...`) + sigv4_interceptor = aws4Interceptor({ + options: { + region: opts.aws_auth.aws_region, + service: 'es' + }, + credentials: { + accessKeyId: opts.aws_auth.aws_access_key_id, + secretAccessKey: opts.aws_auth.aws_access_secret_key, + sessionToken: opts.aws_auth.aws_access_session_token, + } + }); + } else { + this._logger.warn(`No credentials provided, did you forget to set OPENSEARCH_PASSWORD or AWS_ACCESS_KEY_ID?`) + } + this._axios = axios.create({ baseURL: opts?.url ?? DEFAULT_URL, - auth: opts?.username !== undefined && opts.password !== undefined - ? { - username: opts.username, - password: opts.password - } - : undefined, + auth, httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), responseType: opts?.responseType, }) + + if (sigv4_interceptor !== undefined) { + this._axios.interceptors.request.use(sigv4_interceptor) + } } async wait_until_available (max_attempts: number = 20, wait_between_attempt_millis: number = 5000): Promise { let attempt = 0 while (true) { attempt += 1 + this._logger.info(`Connecting to ${this._opts?.url} ... (${attempt}/${max_attempts})`) try { const info = await this.get('/') if (this._opts?.responseType == 'arraybuffer') { @@ -101,9 +182,14 @@ export class OpenSearchHttpClient { } } catch (e) { if (axios.isAxiosError(e)) { + this._logger.warn(`Error connecting to ${this._opts?.url}: (${e.message})`) if (e.response?.status == 401 || e.code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') { - throw e + throw e.message + } else if (e.response?.status == 403 || e.code === 'ERR_BAD_REQUEST') { + throw e.message } + } else { + this._logger.warn(`Error connecting to ${this._opts?.url}: (${typeof (e)})`) } if (attempt >= max_attempts) { throw e diff --git a/tools/src/dump-cluster-spec/dump-cluster-spec.ts b/tools/src/dump-cluster-spec/dump-cluster-spec.ts index 1e65670e8..02dd77972 100644 --- a/tools/src/dump-cluster-spec/dump-cluster-spec.ts +++ b/tools/src/dump-cluster-spec/dump-cluster-spec.ts @@ -48,7 +48,7 @@ const command = new Command() const opts = command.opts() -main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ opensearchResponseType: undefined, ...opts }) }) +main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ responseType: undefined, ...opts }) }) .catch(e => { if (e instanceof Error) { console.error(`ERROR: ${e.stack}`) diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index a05800b37..134888d7f 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -11,6 +11,11 @@ import { Logger, LogLevel } from '../Logger' import TestRunner from './TestRunner' import { Command, Option } from '@commander-js/extra-typings' import { + AWS_ACCESS_KEY_ID_OPTION, + AWS_REGION_OPTION, + AWS_SECRET_ACCESS_KEY_OPTION, + AWS_SERVICE_OPTION, + AWS_SESSION_TOKEN_OPTION, get_opensearch_opts_from_cli, OPENSEARCH_INSECURE_OPTION, OPENSEARCH_PASSWORD_OPTION, @@ -46,6 +51,11 @@ const command = new Command() .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) .addOption(OPENSEARCH_INSECURE_OPTION) + .addOption(AWS_ACCESS_KEY_ID_OPTION) + .addOption(AWS_SECRET_ACCESS_KEY_OPTION) + .addOption(AWS_SESSION_TOKEN_OPTION) + .addOption(AWS_REGION_OPTION) + .addOption(AWS_SERVICE_OPTION) .addOption(new Option('--coverage ', 'path to write test coverage results to')) .allowExcessArguments(false) .parse() @@ -54,7 +64,7 @@ const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) -const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts })) +const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) diff --git a/tools/tests/tester/OpenSearchHttpClient.test.ts b/tools/tests/tester/OpenSearchHttpClient.test.ts new file mode 100644 index 000000000..e2687e7fd --- /dev/null +++ b/tools/tests/tester/OpenSearchHttpClient.test.ts @@ -0,0 +1,66 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import axios from "axios"; +import { OpenSearchHttpClient } from "OpenSearchHttpClient" + +jest.mock('axios') + +describe('OpenSearchHttpClient', () => { + let mocked_axios: jest.Mocked = axios as jest.Mocked + + beforeEach(() => { + mocked_axios.create.mockReturnThis() + mocked_axios.interceptors.request.use = jest.fn().mockReturnValue({ headers: {} }) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + it('uses password authentication', () => { + new OpenSearchHttpClient({ + url: 'https://localhost:9200', + basic_auth: { + username: 'admin', + password: 'password' + } + }) + + expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ + auth: { + username: 'admin', + password: 'password' + }, + baseURL: 'https://localhost:9200' + }) + + expect(mocked_axios.interceptors.request.use).not.toHaveBeenCalled() + }) + + it('assigns a request interceptor with SigV4 authentication', () => { + new OpenSearchHttpClient({ + url: 'https://localhost:9200', + aws_auth: { + aws_access_key_id: 'key id', + aws_access_secret_key: 'secret key', + aws_access_session_token: 'session token', + aws_region: 'us-west-2', + aws_service: 'aoss' + } + }) + + expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ + auth: undefined, + baseURL: 'https://localhost:9200' + }) + + expect(mocked_axios.interceptors.request.use).toHaveBeenCalled() + }) +}) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index dd6758d2a..9e4cf2d46 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -41,8 +41,10 @@ export function construct_tester_components (spec_path: string): { const operation_locator = new OperationLocator(specification) const opensearch_http_client = new OpenSearchHttpClient({ insecure: true, - username: process.env.OPENSEARCH_USERNAME ?? 'admin', - password: process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!', + basic_auth: { + username: process.env.OPENSEARCH_USERNAME ?? 'admin', + password: process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + }, responseType: 'arraybuffer' }) const chapter_reader = new ChapterReader(opensearch_http_client, logger) From b50eb79ce746b4adf5988b99abdc50d875e82035 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 16:56:22 -0400 Subject: [PATCH 13/31] Run tests against stable OpenSearch 2.16. (#484) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 12 ++++++------ .github/workflows/test-tools-integ.yml | 2 +- spec/_info.yaml | 2 +- tools/tests/tester/fixtures/evals/passed.yaml | 2 +- .../tests/tester/fixtures/evals/skipped/semver.yaml | 2 +- tools/tests/tester/helpers.ts | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 3fa404483..b0014a322 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -27,17 +27,17 @@ jobs: admin_password: admin - version: 2.0.0 admin_password: admin - - version: 2.15.0 - - version: 2.15.0 + - version: 2.16.0 + - version: 2.16.0 tests: plugins/index_state_management - - version: 2.15.0 - tests: snapshot - version: 2.16.0 + tests: snapshot + - version: 2.17.0 hub: opensearchstaging - ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e' + ref: '@sha256:6398c27d7560626ed6b0ba28b3d6b20b7f00c6d94abf45ad3a820f8eeb3d61a3' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b' + ref: '@sha256:101681eea630393f8caf5987dd023a975a9656b63090a07bfdfe6ad2f73f0640' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index 660a55e7e..5bb9ac5d8 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -24,7 +24,7 @@ jobs: test: runs-on: ubuntu-latest env: - OPENSEARCH_VERSION: 2.15.0 + OPENSEARCH_VERSION: 2.16.0 OPENSEARCH_PASSWORD: myStrongPassword123! OPENSEARCH_URL: https://localhost:9200 steps: diff --git a/spec/_info.yaml b/spec/_info.yaml index 2454ecb55..d8d63d088 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -2,4 +2,4 @@ $schema: ./json_schemas/_info.schema.yaml title: OpenSearch API Specification version: 1.0.0 -x-api-version: 2.15.0 +x-api-version: 2.16.0 diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index bb74087e8..12c1b2c32 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -175,7 +175,7 @@ chapters: - title: This GET /_cat/health should be skipped (> 2.999.0). overall: result: SKIPPED - message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. + message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/evals/skipped/semver.yaml b/tools/tests/tester/fixtures/evals/skipped/semver.yaml index b39460dcb..30709742d 100644 --- a/tools/tests/tester/fixtures/evals/skipped/semver.yaml +++ b/tools/tests/tester/fixtures/evals/skipped/semver.yaml @@ -3,5 +3,5 @@ full_path: tools/tests/tester/fixtures/stories/skipped/semver.yaml result: SKIPPED description: This story should be skipped because of version. -message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. +message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 9e4cf2d46..ce11b6ad8 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.15.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) } From 6cdd8b7e86e2a2a08c07929f049d3ed17231b646 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 17:04:32 -0400 Subject: [PATCH 14/31] Fix: durations are not always integers. (#479) Signed-off-by: dblock --- CHANGELOG.md | 3 ++- spec/schemas/_common.yaml | 2 +- tools/tests/tester/fixtures/specs/excerpt.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5590437d..a0ecf4f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `/_mapping` with `index` in query ([#385](https://github.com/opensearch-project/opensearch-api-specification/pull/385)) - Fixed duplicate `/_nodes/{node_id}` path ([#416](https://github.com/opensearch-project/opensearch-api-specification/pull/416)) - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430)) -- Fixed `_update_by_query` with a simple term ([451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) +- Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) +- Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) ### Security diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 1c20ad16b..5b645eed4 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -38,7 +38,7 @@ components: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value. - pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$ + pattern: ^([0-9\.]+)(?:d|h|m|s|ms|micros|nanos)$ type: string Metadata: type: object diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml index 64ad01d39..244e7e466 100644 --- a/tools/tests/tester/fixtures/specs/excerpt.yaml +++ b/tools/tests/tester/fixtures/specs/excerpt.yaml @@ -286,5 +286,5 @@ components: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value. - pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$ + pattern: ^([0-9\.]+)(?:d|h|m|s|ms|micros|nanos)$ type: string \ No newline at end of file From ecf39215e18d9edeba1d048b2ba22a172844ae6c Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 17:06:34 -0400 Subject: [PATCH 15/31] Fix: refresh index to get a consistent total. (#480) Signed-off-by: dblock --- tests/default/_core/reindex.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/default/_core/reindex.yaml b/tests/default/_core/reindex.yaml index 86d4fccb4..71567b7a9 100644 --- a/tests/default/_core/reindex.yaml +++ b/tests/default/_core/reindex.yaml @@ -39,6 +39,8 @@ chapters: - synopsis: Reindex a subset of documents (match). path: /_reindex method: POST + parameters: + refresh: true request: payload: source: @@ -82,7 +84,7 @@ chapters: response: status: 200 payload: - total: 1 + total: 2 - synopsis: Reindex only unique documents. path: /_reindex method: POST From d7e6971222d70f0219a01ba2ebb4941092d680de Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 11 Aug 2024 18:17:59 -0400 Subject: [PATCH 16/31] Double the RAM in the test container. (#486) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b0014a322..72a8d7601 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -47,7 +47,7 @@ jobs: OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }} OPENSEARCH_VERSION: ${{ matrix.entry.version }} OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} - OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms2g -Xmx2g + OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g steps: - name: Checkout Repo From 56a5a224562d465f77c6e1712a266d1d11ba8a5c Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Sun, 11 Aug 2024 18:42:31 -0400 Subject: [PATCH 17/31] Document fielddata.yaml fail, ignore vim swp files (#473) * Document fielddata.yaml fail, ignore vim swp files * Ignore test files that start with . so vim swp files are ignored * fielddata.yaml test files if opensearch cluster was stopped with docker kill instead of docker stop, document the issue * upgrade remaining actions/checkout@v3 to v4 to stop warnings Signed-off-by: Andy Wick * fixed FAILED placment Signed-off-by: Andy Wick * switch to only processing .yaml files Signed-off-by: Andy Wick * Remove EXCLUDED_FILES instead only process .yaml Signed-off-by: Andy Wick --------- Signed-off-by: Andy Wick --- .github/workflows/changelog.yml | 2 +- .github/workflows/test-spec.yml | 2 +- TESTING_GUIDE.md | 4 ++++ tools/src/tester/TestRunner.ts | 11 ++++------- .../tester/fixtures/stories/ignore.wrong.extension | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 tools/tests/tester/fixtures/stories/ignore.wrong.extension diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 23a1e0502..c6994060d 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -7,7 +7,7 @@ jobs: verify-changelog: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 72a8d7601..d9ed81aea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -95,7 +95,7 @@ jobs: if: github.event_name == 'pull_request' needs: test-opensearch-spec steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download Spec Coverage Data uses: actions/download-artifact@v4 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 5c066549e..61f6c2828 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -6,6 +6,7 @@ - [Common Errors](#common-errors) - [401 Unauthorized](#401-unauthorized) - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + - [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) - [Writing Spec Tests](#writing-spec-tests) - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) @@ -85,6 +86,9 @@ curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_clu ' ``` +#### FAILED Cat with a json response (from security-analytics). +The cluster is not loading plugins correctly, maybe it was stopped using `docker kill` instead of `docker stop`. Recreating the cluster should fix the issue: `docker compose up --force-recreate -d`. + ## Writing Spec Tests The spec tests reside in the [tests/](tests) directory. Tests are organized in suites ([default](tests/default/), etc.), and subsequently in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/default/indices/index.yaml](tests/default/indices/index.yaml) (for `/{index}`), and [tests/default/indices/doc.yaml](tests/default/indices/doc.yaml) (for `/{index}/_doc`). diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index fdde53791..0c0f3c4ff 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -20,10 +20,6 @@ import { OpenSearchHttpClient } from 'OpenSearchHttpClient' import * as ansi from './Ansi' import _ from 'lodash' -const EXCLUDED_FILES = [ - 'docker-compose.yml' -] - export default class TestRunner { private readonly _http_client: OpenSearchHttpClient private readonly _story_validator: StoryValidator @@ -69,6 +65,9 @@ export default class TestRunner { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` if (fs.statSync(path).isFile()) { + if (!path.endsWith('.yaml')) { + return [] + } const story: Story = read_yaml(path) return [{ display_path: next_prefix === '' ? basename(path) : next_prefix, @@ -77,9 +76,7 @@ export default class TestRunner { }] } else { return _.compact(fs.readdirSync(path).flatMap(next_file => { - if (!EXCLUDED_FILES.includes(next_file)) { - return this.#collect_story_files(path, next_file, next_prefix) - } + return this.#collect_story_files(path, next_file, next_prefix) })) } } diff --git a/tools/tests/tester/fixtures/stories/ignore.wrong.extension b/tools/tests/tester/fixtures/stories/ignore.wrong.extension new file mode 100644 index 000000000..a649676ae --- /dev/null +++ b/tools/tests/tester/fixtures/stories/ignore.wrong.extension @@ -0,0 +1 @@ +Not a .yaml file, should be ignored From 00711ea55e7317840a6617bae872997ca4a14e63 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 11 Aug 2024 19:20:56 -0400 Subject: [PATCH 18/31] Allow overwriting the Authorization header. (#488) Signed-off-by: dblock --- package-lock.json | 49 +++++++++++++++ package.json | 1 + tools/src/OpenSearchHttpClient.ts | 20 +++--- .../tests/tester/OpenSearchHttpClient.test.ts | 62 +++++++++++-------- 4 files changed, 98 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index e1156510e..00b73c87a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "ajv-formats": "^3.0.1", "aws4-axios": "^3.3.7", "axios": "^1.7.1", + "axios-mock-adapter": "^2.0.0", "cbor": "^9.0.2", "commander": "^12.0.0", "eslint": "^8.57.0", @@ -3935,6 +3936,18 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-mock-adapter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.0.0.tgz", + "integrity": "sha512-D/K0J5Zm6KvaMTnsWrBQZWLzKN9GxUFZEa0mx2qeEHXDeTugCoplWehy8y36dj5vuSjhe1u/Dol8cZ8lzzmDew==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + }, + "peerDependencies": { + "axios": ">= 0.17.0" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -6423,6 +6436,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", @@ -12467,6 +12502,15 @@ "proxy-from-env": "^1.1.0" } }, + "axios-mock-adapter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.0.0.tgz", + "integrity": "sha512-D/K0J5Zm6KvaMTnsWrBQZWLzKN9GxUFZEa0mx2qeEHXDeTugCoplWehy8y36dj5vuSjhe1u/Dol8cZ8lzzmDew==", + "requires": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + } + }, "babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -14192,6 +14236,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, "is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", diff --git a/package.json b/package.json index 22d798e13..5ffe41341 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@types/titlecase": "^1.1.2", "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^6.21.0", + "axios-mock-adapter": "^2.0.0", "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index f81656074..afbb7d5fa 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -133,18 +133,23 @@ export class OpenSearchHttpClient { this._opts = opts this._logger = opts?.logger ?? new Logger() - let auth = undefined - let sigv4_interceptor = undefined + let auth_middleware = undefined if (opts?.basic_auth !== undefined) { this._logger.info(`Authenticating with ${opts.basic_auth.username} ...`) - auth = opts.basic_auth + auth_middleware = ((request: any): any => { + if (request.headers.Authorization === undefined) { + const base64 = Buffer.from(`${opts.basic_auth?.username}:${opts.basic_auth?.password}`, 'utf8').toString('base64'); + request.headers.Authorization = `Basic ${base64}` + } + return request + }) } else if (opts?.aws_auth !== undefined) { this._logger.info(`Authenticating using SigV4 with ${opts.aws_auth.aws_access_key_id} (${opts.aws_auth.aws_region}) ...`) - sigv4_interceptor = aws4Interceptor({ + auth_middleware = aws4Interceptor({ options: { region: opts.aws_auth.aws_region, - service: 'es' + service: opts.aws_auth.aws_service }, credentials: { accessKeyId: opts.aws_auth.aws_access_key_id, @@ -158,13 +163,12 @@ export class OpenSearchHttpClient { this._axios = axios.create({ baseURL: opts?.url ?? DEFAULT_URL, - auth, httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), responseType: opts?.responseType, }) - if (sigv4_interceptor !== undefined) { - this._axios.interceptors.request.use(sigv4_interceptor) + if (auth_middleware !== undefined) { + this._axios.interceptors.request.use(auth_middleware) } } diff --git a/tools/tests/tester/OpenSearchHttpClient.test.ts b/tools/tests/tester/OpenSearchHttpClient.test.ts index e2687e7fd..85c62af05 100644 --- a/tools/tests/tester/OpenSearchHttpClient.test.ts +++ b/tools/tests/tester/OpenSearchHttpClient.test.ts @@ -9,58 +9,68 @@ import axios from "axios"; import { OpenSearchHttpClient } from "OpenSearchHttpClient" - -jest.mock('axios') +import AxiosMockAdapter from "axios-mock-adapter"; describe('OpenSearchHttpClient', () => { - let mocked_axios: jest.Mocked = axios as jest.Mocked + var mock = new AxiosMockAdapter(axios) - beforeEach(() => { - mocked_axios.create.mockReturnThis() - mocked_axios.interceptors.request.use = jest.fn().mockReturnValue({ headers: {} }) + afterEach(() => { + mock.reset() }) - afterEach(() => { - jest.clearAllMocks() + it('adds a Basic auth header', async () => { + let client = new OpenSearchHttpClient({ + url: 'https://localhost:9200', + basic_auth: { + username: 'u', + password: 'p' + } + }) + + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toMatch(/^Basic /) + return [200, { called: true }] + }) + + expect((await client.get('/')).data).toEqual({ called: true }) }) - it('uses password authentication', () => { - new OpenSearchHttpClient({ + it('allows to overwrite Authorization', async () => { + let client = new OpenSearchHttpClient({ url: 'https://localhost:9200', basic_auth: { - username: 'admin', - password: 'password' + username: 'u', + password: 'p' } }) - expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ - auth: { - username: 'admin', - password: 'password' - }, - baseURL: 'https://localhost:9200' + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toEqual('custom') + return [200, { called: true }] }) - expect(mocked_axios.interceptors.request.use).not.toHaveBeenCalled() + expect((await client.get('/', { headers: { Authorization: 'custom' } })).data).toEqual({ called: true }) }) - it('assigns a request interceptor with SigV4 authentication', () => { - new OpenSearchHttpClient({ + it('adds a Sigv4 header', async () => { + let client = new OpenSearchHttpClient({ url: 'https://localhost:9200', aws_auth: { aws_access_key_id: 'key id', aws_access_secret_key: 'secret key', aws_access_session_token: 'session token', - aws_region: 'us-west-2', + aws_region: 'us-west-42', aws_service: 'aoss' } }) - expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ - auth: undefined, - baseURL: 'https://localhost:9200' + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toMatch( + /^AWS4-HMAC-SHA256 Credential=key id\/\d*\/us-west-42\/aoss\/aws4_request/ + ) + return [200, { called: true }] }) - expect(mocked_axios.interceptors.request.use).toHaveBeenCalled() + expect((await client.get('/')).data).toEqual({ called: true }) }) }) From 644a1a5b5f79ec0803ded1d408961a01e6deaccf Mon Sep 17 00:00:00 2001 From: Alen <78027095+aabeshov@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:58:36 +0500 Subject: [PATCH 19/31] Observability namespace (#474) * Observability api and query api Signed-off-by: alen_abeshov * Fixing wrong linting Signed-off-by: alen_abeshov * added tests Signed-off-by: alen_abeshov * removed docker file Signed-off-by: alen_abeshov * Added tests Signed-off-by: alen_abeshov * Removed spaces, added exclude file Signed-off-by: alen_abeshov * Changed lycheeignore Signed-off-by: alen_abeshov * Fixing path Signed-off-by: alen_abeshov * Change URL Signed-off-by: alen_abeshov * add verbosity Signed-off-by: alen_abeshov * Checking version Signed-off-by: alen_abeshov * changin operation version Signed-off-by: alen_abeshov * Replacing http Signed-off-by: alen_abeshov * Changed desc Signed-off-by: alen_abeshov * Changed desc Signed-off-by: alen_abeshov * Fixing bug Signed-off-by: alen_abeshov --------- Signed-off-by: alen_abeshov Signed-off-by: Alen <78027095+aabeshov@users.noreply.github.com> --- .cspell | 3 +- .lycheeignore | 2 +- CHANGELOG.md | 1 + spec/namespaces/observability.yaml | 224 +++++++++++++++ spec/namespaces/query.yaml | 140 ++++++++++ spec/schemas/observability._common.yaml | 255 ++++++++++++++++++ spec/schemas/query._common.yaml | 131 +++++++++ .../default/observability/observability.yaml | 155 +++++++++++ tests/default/query/datasources.yaml | 95 +++++++ 9 files changed, 1004 insertions(+), 2 deletions(-) create mode 100644 spec/namespaces/observability.yaml create mode 100644 spec/namespaces/query.yaml create mode 100644 spec/schemas/observability._common.yaml create mode 100644 spec/schemas/query._common.yaml create mode 100644 tests/default/observability/observability.yaml create mode 100644 tests/default/query/datasources.yaml diff --git a/.cspell b/.cspell index 1542876a8..900b6c40a 100644 --- a/.cspell +++ b/.cspell @@ -185,4 +185,5 @@ urldecode vectory whoamiprotected wordnet -Yrtsd \ No newline at end of file +Yrtsd +localstats \ No newline at end of file diff --git a/.lycheeignore b/.lycheeignore index f437ebb39..c859dd4da 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1 +1 @@ -https://localhost:9200* +https://localhost:* diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ecf4f75..a941ea033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) +- Added `observability` namespace API specifications ([#474](https://github.com/opensearch-project/opensearch-api-specification/pull/474)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) diff --git a/spec/namespaces/observability.yaml b/spec/namespaces/observability.yaml new file mode 100644 index 000000000..a8553dbe1 --- /dev/null +++ b/spec/namespaces/observability.yaml @@ -0,0 +1,224 @@ +openapi: 3.1.0 +info: + title: OpenSearch Observability Object API + description: API for retrieving and managing Observability Objects. + version: 1.0.0 +paths: + /_plugins/_observability/_local/stats: + get: + operationId: observability.get_localstats.0 + x-operation-group: observability.get_localstats + x-version-added: '1.1' + description: Retrieves Local Stats of all observability objects. + responses: + '200': + $ref: '#/components/responses/observability.get_localstats@200' + /_plugins/_observability/object: + get: + operationId: observability.list_objects.0 + x-operation-group: observability.list_objects + x-version-added: '1.1' + description: Retrieves list of all observability objects. + responses: + '200': + $ref: '#/components/responses/observability.list_objects@200' + post: + operationId: observability.create_object.0 + x-operation-group: observability.create_object + x-version-added: '1.1' + description: Creates a new observability object. + requestBody: + $ref: '#/components/requestBodies/observability.create_object' + responses: + '200': + $ref: '#/components/responses/observability.create_object@200' + delete: + operationId: observability.delete_objects.0 + x-operation-group: observability.delete_objects + x-version-added: '1.1' + description: Deletes specific observability objects specified by ID or a list of IDs. + parameters: + - $ref: '#/components/parameters/observability.delete_objects::query.objectId' + - $ref: '#/components/parameters/observability.delete_objects::query.objectIdList' + responses: + '200': + $ref: '#/components/responses/observability.delete_objects@200' + '404': + $ref: '#/components/responses/observability.delete_objects@404' + /_plugins/_observability/object/{object_id}: + get: + operationId: observability.get_object.0 + x-operation-group: observability.get_object + x-version-added: '1.1' + description: Retrieves specific observability object specified by ID. + parameters: + - $ref: '#/components/parameters/observability.get_object::path.object_id' + responses: + '200': + $ref: '#/components/responses/observability.get_object@200' + '404': + $ref: '#/components/responses/observability.get_object@404' + put: + operationId: observability.update_object.0 + x-operation-group: observability.update_object + x-version-added: '1.1' + description: Updates an existing observability object. + parameters: + - $ref: '#/components/parameters/observability.update_object::path.object_id' + requestBody: + $ref: '#/components/requestBodies/observability.update_object' + responses: + '200': + $ref: '#/components/responses/observability.update_object@200' + '404': + $ref: '#/components/responses/observability.update_object@404' + delete: + operationId: observability.delete_object.0 + x-operation-group: observability.delete_object + x-version-added: '1.1' + description: Deletes specific observability object specified by ID. + parameters: + - $ref: '#/components/parameters/observability.delete_object::path.object_id' + responses: + '200': + $ref: '#/components/responses/observability.delete_object@200' + '404': + $ref: '#/components/responses/observability.delete_object@404' +components: + requestBodies: + observability.create_object: + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObject' + observability.update_object: + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObject' + responses: + observability.list_objects@200: + description: Successful response of retrieving all Observability Objects. + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObjectList' + observability.create_object@200: + description: Created + content: + application/json: + schema: + type: object + properties: + objectId: + type: string + observability.get_object@200: + description: Successful response of retrieving specific Observability Object. + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObjectList' + observability.get_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.update_object@200: + description: Updated + content: + application/json: + schema: + type: object + properties: + objectId: + type: string + observability.update_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.delete_object@200: + description: Deleted + content: + application/json: + schema: + type: object + properties: + deleteResponseList: + type: object + additionalProperties: + type: string + example: OK + observability.delete_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.delete_objects@200: + description: Deleted + content: + application/json: + schema: + type: object + properties: + deleteResponseList: + type: object + additionalProperties: + type: string + example: OK + observability.delete_objects@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.get_localstats@200: + description: Retrieves + content: + application/json: + schema: + type: string + parameters: + observability.get_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.update_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.delete_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.delete_objects::query.objectId: + in: query + name: objectId + description: The ID of a single Observability Object to delete. + required: false + schema: + type: string + style: form + observability.delete_objects::query.objectIdList: + in: query + name: objectIdList + description: A comma-separated list of Observability Object IDs to delete. + required: false + schema: + type: string + style: form diff --git a/spec/namespaces/query.yaml b/spec/namespaces/query.yaml new file mode 100644 index 000000000..b055f4666 --- /dev/null +++ b/spec/namespaces/query.yaml @@ -0,0 +1,140 @@ +openapi: 3.1.0 +info: + title: OpenSearch Query Datasources API + description: OpenSearch Query Datasources API. + version: 2.7.0 +paths: + /_plugins/_query/_datasources: + get: + operationId: query.datasources_list.0 + x-operation-group: query.datasources_list + x-version-added: '2.7' + description: Retrieves list of all datasources. + responses: + '200': + $ref: '#/components/responses/query.datasources_list@200' + post: + operationId: query.datasources_create.0 + x-operation-group: query.datasources_create + x-version-added: '2.7' + description: Creates a new query datasource. + requestBody: + $ref: '#/components/requestBodies/query.datasources_create' + responses: + '201': + $ref: '#/components/responses/query.datasources_create@201' + put: + operationId: query.datasources_update.0 + x-operation-group: query.datasources_update + x-version-added: '2.7' + description: Updates an existing query datasource. + requestBody: + $ref: '#/components/requestBodies/query.datasources_update' + responses: + '200': + $ref: '#/components/responses/query.datasources_update@200' + '404': + $ref: '#/components/responses/query.datasources_update@404' + /_plugins/_query/_datasources/{datasource_name}: + get: + operationId: query.datasource_retrieve.0 + x-operation-group: query.datasource_retrieve + x-version-added: '2.7' + description: Retrieves specific datasource specified by name. + parameters: + - $ref: '#/components/parameters/query.datasource_retrieve::path.datasource_name' + responses: + '200': + $ref: '#/components/responses/query.datasource_retrieve@200' + '404': + $ref: '#/components/responses/query.datasource_retrieve@404' + delete: + operationId: query.datasource_delete.0 + x-operation-group: query.datasource_delete + x-version-added: '2.7' + description: Deletes specific datasource specified by name. + parameters: + - $ref: '#/components/parameters/query.datasource_delete::path.datasource_name' + responses: + '204': + $ref: '#/components/responses/query.datasource_delete@204' + '404': + $ref: '#/components/responses/query.datasource_delete@404' + +components: + requestBodies: + query.datasources_create: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSource' + query.datasources_update: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSource' + responses: + query.datasources_list@200: + description: Successful response of retrieving all Data Sources. + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceList' + query.datasources_create@201: + description: Created + content: + application/json: + schema: + type: string + query.datasources_update@200: + description: Updated + content: + application/json: + schema: + type: string + query.datasources_update@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + query.datasource_retrieve@200: + description: Successful response of retrieving Data Source. + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceRetrieve' + query.datasource_retrieve@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + query.datasource_delete@204: + description: No Content + content: + application/json: + schema: + type: object + properties: { } + query.datasource_delete@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + parameters: + query.datasource_delete::path.datasource_name: + name: datasource_name + in: path + description: The Name of the DataSource to delete. + schema: + type: string + required: true + query.datasource_retrieve::path.datasource_name: + name: datasource_name + in: path + description: The Name of the DataSource to retrieve. + schema: + type: string + required: true diff --git a/spec/schemas/observability._common.yaml b/spec/schemas/observability._common.yaml new file mode 100644 index 000000000..fc8018f55 --- /dev/null +++ b/spec/schemas/observability._common.yaml @@ -0,0 +1,255 @@ +openapi: 3.1.0 +info: + title: Schemas for OpenSearch Observability Object API + description: Schemas for OpenSearch Observability Object API + version: 1.0.0 +paths: {} +components: + schemas: + ObservabilityObjectList: + type: object + properties: + startIndex: + type: integer + totalHits: + type: integer + totalHitRelation: + type: string + observabilityObjectList: + type: array + items: + $ref: '#/components/schemas/ObservabilityObject' + required: + - observabilityObjectList + - startIndex + - totalHitRelation + - totalHits + + ObservabilityObject: + type: object + properties: + objectId: + type: string + lastUpdatedTimeMs: + type: integer + createdTimeMs: + type: integer + tenant: + type: string + operationalPanel: + $ref: '#/components/schemas/OperationalPanel' + savedVisualization: + $ref: '#/components/schemas/SavedVisualization' + savedQuery: + $ref: '#/components/schemas/SavedQuery' + required: + - objectId + - tenant + + OperationalPanel: + type: object + properties: + name: + type: string + visualizations: + type: array + items: + $ref: '#/components/schemas/Visualization' + timeRange: + $ref: '#/components/schemas/TimeRange' + queryFilter: + $ref: '#/components/schemas/QueryFilter' + applicationId: + type: string + required: + - applicationId + - name + - queryFilter + - timeRange + - visualizations + + Visualization: + type: object + properties: + id: + type: string + savedVisualizationId: + type: string + x: + type: integer + y: + type: integer + w: + type: integer + h: + type: integer + required: + - h + - id + - savedVisualizationId + - w + - x + - y + + TimeRange: + type: object + properties: + to: + type: string + from: + type: string + required: + - from + - to + + QueryFilter: + type: object + properties: + query: + type: string + language: + type: string + required: + - language + - query + + SavedVisualization: + type: object + properties: + name: + type: string + description: + type: string + query: + type: string + type: + type: string + selected_date_range: + $ref: '#/components/schemas/SelectedDateRange' + selected_timestamp: + $ref: '#/components/schemas/SelectedTimestamp' + selected_fields: + $ref: '#/components/schemas/SelectedFields' + required: + - description + - name + - query + - selected_date_range + - selected_fields + - selected_timestamp + - type + + SelectedDateRange: + type: object + properties: + start: + type: string + end: + type: string + text: + type: string + required: + - end + - start + - text + + SelectedTimestamp: + type: object + properties: + name: + type: string + type: + type: string + required: + - name + - type + + SelectedFields: + type: object + properties: + text: + type: string + tokens: + type: array + items: + $ref: '#/components/schemas/Token' + required: + - text + - tokens + + Token: + type: object + properties: + name: + type: string + type: + type: string + required: + - name + - type + + SavedQuery: + type: object + properties: + name: + type: string + description: + type: string + query: + type: string + selected_date_range: + $ref: '#/components/schemas/SelectedDateRange' + selected_timestamp: + $ref: '#/components/schemas/SelectedTimestamp' + selected_fields: + $ref: '#/components/schemas/SelectedFields' + required: + - description + - name + - query + - selected_date_range + - selected_fields + - selected_timestamp + + NotFoundResponse: + type: object + properties: + error: + $ref: '#/components/schemas/ErrorResponse' + status: + type: integer + example: 404 + required: + - error + - status + + ErrorResponse: + type: object + properties: + root_cause: + type: array + items: + $ref: '#/components/schemas/RootCause' + type: + type: string + example: status_exception + reason: + type: string + example: 'ObservabilityObject {objectId} not found' + required: + - reason + - root_cause + - type + + RootCause: + type: object + properties: + type: + type: string + example: status_exception + reason: + type: string + example: 'ObservabilityObject {objectId} not found' + required: + - reason + - type diff --git a/spec/schemas/query._common.yaml b/spec/schemas/query._common.yaml new file mode 100644 index 000000000..080c3afa3 --- /dev/null +++ b/spec/schemas/query._common.yaml @@ -0,0 +1,131 @@ +openapi: 3.1.0 +info: + title: Schemas for OpenSearch Query Datasources API + description: Schemas for OpenSearch Query Datasources API + version: 1.0.0 +paths: {} +components: + schemas: + DataSourceList: + type: array + items: + $ref: '#/components/schemas/DataSource' + + DataSource: + type: object + properties: + name: + type: string + description: + type: string + connector: + type: string + allowedRoles: + type: array + items: + type: string + properties: + type: object + additionalProperties: true + resultIndex: + type: string + status: + type: string + configuration: + $ref: '#/components/schemas/DataSourceConfiguration' + required: + - connector + - name + - properties + - resultIndex + - status + + DataSourceConfiguration: + type: object + properties: + endpoint: + type: string + credentials: + $ref: '#/components/schemas/Credentials' + required: + - credentials + - endpoint + + Credentials: + type: object + properties: + username: + type: string + password: + type: string + required: + - password + - username + + DataSourceNotFound: + type: object + properties: + error: + $ref: '#/components/schemas/ErrorResponse' + required: + - error + + ErrorResponse: + type: object + properties: + root_cause: + type: array + items: + $ref: '#/components/schemas/RootCause' + type: + type: string + example: status_exception + reason: + type: string + example: DataSource not found + required: + - reason + - root_cause + - type + + RootCause: + type: object + properties: + type: + type: string + example: status_exception + reason: + type: string + example: DataSource not found + required: + - reason + - type + + DataSourceRetrieve: + type: object + properties: + name: + type: string + description: + type: string + connector: + type: string + allowedRoles: + type: array + items: + type: string + properties: + type: object + additionalProperties: true + resultIndex: + type: string + status: + type: string + configuration: + $ref: '#/components/schemas/DataSourceConfiguration' + required: + - connector + - name + - properties + - resultIndex + - status diff --git a/tests/default/observability/observability.yaml b/tests/default/observability/observability.yaml new file mode 100644 index 000000000..f9e37461c --- /dev/null +++ b/tests/default/observability/observability.yaml @@ -0,0 +1,155 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test various operations of the OpenSearch Observability Object API. + +prologues: + - path: /_plugins/_observability/object/{object_id} + method: DELETE + parameters: + object_id: test_object + status: [200, 404] + - path: /_plugins/_observability/object + method: POST + request: + payload: + objectId: test_object + operationalPanel: + name: test_panel + visualizations: [] + timeRange: + from: now-1h + to: now + queryFilter: + query: '' + language: ppl + applicationId: test_app + savedVisualization: + name: viz1 + description: desc1 + query: '' + type: line + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + savedQuery: + name: query1 + description: desc1 + query: '' + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + status: [200] +chapters: + - synopsis: Retrieve specific Observability object after creation. + path: /_plugins/_observability/object/{object_id} + id: observatory_object + method: GET + parameters: + object_id: test_object + response: + status: 200 + payload: + startIndex: 0 + totalHits: 1 + totalHitRelation: eq + observabilityObjectList: [] + - synopsis: Update specific Observability object. + path: /_plugins/_observability/object/{object_id} + method: PUT + parameters: + object_id: test_object + request: + payload: + objectId: test_object + tenant: '' + operationalPanel: + name: updated_test_panel + visualizations: [] + timeRange: + from: now-1h + to: now + queryFilter: + query: '' + language: ppl + applicationId: test_app + savedVisualization: + name: '[Logs] Count total requests by tags' + description: '' + query: 'source = opensearch_dashboards_sample_data_logs | stats count() by tags' + type: bar + selected_date_range: + start: now/y + end: now + text: '' + selected_timestamp: + name: timestamp + type: timestamp + selected_fields: + text: '' + tokens: [] + savedQuery: + name: query1 + description: desc1 + query: '' + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + response: + status: 200 + payload: + objectId: test_object + - synopsis: Retrieve specific Observability object after update. + path: /_plugins/_observability/object/{object_id} + method: GET + parameters: + object_id: test_object + response: + status: 200 + payload: + startIndex: 0 + totalHits: 1 + totalHitRelation: eq + observabilityObjectList: [] + - synopsis: Retrieve list of Observability objects. + path: /_plugins/_observability/object + method: GET + response: + status: 200 + payload: + startIndex: 0 + totalHits: 0 + totalHitRelation: eq + observabilityObjectList: [] +epilogues: + - path: /_plugins/_observability/object/{object_id} + method: DELETE + parameters: + object_id: test_object + status: [200, 404] \ No newline at end of file diff --git a/tests/default/query/datasources.yaml b/tests/default/query/datasources.yaml new file mode 100644 index 000000000..35c9fea04 --- /dev/null +++ b/tests/default/query/datasources.yaml @@ -0,0 +1,95 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test various operations of the OpenSearch Query Datasources API. +version: '>=2.7' + +prologues: + - path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + status: [204, 404] +chapters: + - synopsis: Create a new query datasource. + path: /_plugins/_query/_datasources + method: POST + request: + payload: + name: test_datasource + description: '' + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9090' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + response: + status: 201 + payload: Created DataSource with name test_datasource + - synopsis: Retrieve the list of all query datasources. + path: /_plugins/_query/_datasources + method: GET + response: + status: 200 + payload: [] + - synopsis: Retrieve a specific query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: GET + parameters: + datasource_name: test_datasource + response: + status: 200 + payload: + name: test_datasource + description: '' + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9090' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + - synopsis: Update an existing query datasource. + path: /_plugins/_query/_datasources + method: PUT + request: + payload: + name: test_datasource + description: Updated description + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9091' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + response: + status: 200 + payload: Updated DataSource with name test_datasource + - synopsis: Retrieve the updated query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: GET + parameters: + datasource_name: test_datasource + response: + status: 200 + payload: + name: test_datasource + description: Updated description + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9091' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + - synopsis: Delete the query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + response: + status: 204 +epilogues: + - path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + status: [204, 404] \ No newline at end of file From 597ed8d73c0b72743ed90fd9694ad9ba511d5be7 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Mon, 12 Aug 2024 22:59:54 +1200 Subject: [PATCH 20/31] Improve indices.stats schemas (#491) * Improve indices.stats schemas Signed-off-by: Thomas Farr * Add changelog Signed-off-by: Thomas Farr * Remove unused schemas Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/namespaces/indices.yaml | 8 +- spec/schemas/_common.yaml | 147 +++++++++--------------- spec/schemas/indices.stats.yaml | 195 +++++++++----------------------- spec/schemas/nodes._common.yaml | 2 +- 5 files changed, 118 insertions(+), 235 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a941ea033..a5821316f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430)) - Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) +- Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) ### Security diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index 6d36982f5..55f0c2999 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2686,14 +2686,14 @@ components: schema: type: object properties: + _shards: + $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' + _all: + $ref: '../schemas/indices.stats.yaml#/components/schemas/AllIndicesStats' indices: type: object additionalProperties: $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' - _shards: - $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' - _all: - $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' required: - _all - _shards diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 5b645eed4..f89bdb9bd 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1250,13 +1250,6 @@ components: reserved_in_bytes: description: A prediction, in bytes, of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. type: number - total_data_set_size: - $ref: '#/components/schemas/ByteSize' - total_data_set_size_in_bytes: - description: |- - Total data set size, in bytes, of all shards assigned to selected nodes. - This includes the size of shards not stored fully on the nodes, such as the cache for partially mounted indices. - type: number required: - reserved_in_bytes - size_in_bytes @@ -1503,25 +1496,25 @@ components: GetStats: type: object properties: - current: + total: type: number - exists_time: + time: $ref: '#/components/schemas/Duration' - exists_time_in_millis: + time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' exists_total: type: number - missing_time: + exists_time: $ref: '#/components/schemas/Duration' - missing_time_in_millis: + exists_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' missing_total: type: number - time: + missing_time: $ref: '#/components/schemas/Duration' - time_in_millis: + missing_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - total: + current: type: number required: - current @@ -1534,40 +1527,39 @@ components: IndexingStats: type: object properties: + index_total: + type: number + index_time: + $ref: '#/components/schemas/Duration' + index_time_in_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' index_current: type: number - delete_current: + index_failed: + type: number + delete_total: type: number delete_time: $ref: '#/components/schemas/Duration' delete_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - delete_total: + delete_current: type: number - doc_status: - $ref: '#/components/schemas/DocStatus' - is_throttled: - type: boolean noop_update_total: type: number + is_throttled: + type: boolean throttle_time: $ref: '#/components/schemas/Duration' throttle_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - index_time: - $ref: '#/components/schemas/Duration' - index_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - index_total: - type: number - index_failed: - type: number + doc_status: + $ref: '#/components/schemas/DocStatus' types: + x-version-removed: '2.0' type: object additionalProperties: $ref: '#/components/schemas/IndexingStats' - write_load: - type: number required: - delete_current - delete_time_in_millis @@ -1684,6 +1676,21 @@ components: SearchStats: type: object properties: + open_contexts: + description: The number of open search contexts. + type: number + query_current: + description: The number of shard query operations that are currently running. + type: number + query_time: + description: The total amount of time for all shard query operations. + $ref: '#/components/schemas/Duration' + query_time_in_millis: + description: The total amount of time for all shard query operations, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + query_total: + description: The total number of shard query operations. + type: number concurrent_query_total: description: The total number of query operations that use concurrent segment search. type: number @@ -1710,39 +1717,6 @@ components: fetch_total: description: The total number of shard fetch operations. type: number - open_contexts: - description: The number of open search contexts. - type: number - point_in_time_total: - description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. - type: number - point_in_time_time: - $ref: '#/components/schemas/Duration' - point_in_time_time_in_millis: - description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - point_in_time_current: - description: The number of shard PIT contexts currently open. - type: number - query_current: - description: The number of shard query operations that are currently running. - type: number - query_time: - description: The total amount of time for all shard query operations. - $ref: '#/components/schemas/Duration' - query_time_in_millis: - description: The total amount of time for all shard query operations, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - query_total: - description: The total number of shard query operations. - type: number - request: - type: object - description: Statistics about coordinator search operations for the node. - additionalProperties: - $ref: '#/components/schemas/RequestStats' - search_idle_reactivate_count_total: - type: number scroll_current: description: The number of shard scroll operations that are currently running. type: number @@ -1755,6 +1729,17 @@ components: scroll_total: description: The total number of shard scroll operations. type: number + point_in_time_total: + description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. + type: number + point_in_time_time: + $ref: '#/components/schemas/Duration' + point_in_time_time_in_millis: + description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + point_in_time_current: + description: The number of shard PIT contexts currently open. + type: number suggest_current: description: The number of shard suggest operations that are currently running. type: number @@ -1767,6 +1752,13 @@ components: suggest_total: description: The total number of shard suggest operations. type: number + search_idle_reactivate_count_total: + type: number + request: + type: object + description: Statistics about coordinator search operations for the node. + additionalProperties: + $ref: '#/components/schemas/RequestStats' groups: type: object additionalProperties: @@ -1833,33 +1825,6 @@ components: - current - total - total_time_in_millis - BulkStats: - type: object - properties: - total_operations: - type: number - total_time: - $ref: '#/components/schemas/Duration' - total_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - total_size: - $ref: '#/components/schemas/ByteSize' - total_size_in_bytes: - type: number - avg_time: - $ref: '#/components/schemas/Duration' - avg_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - avg_size: - $ref: '#/components/schemas/ByteSize' - avg_size_in_bytes: - type: number - required: - - avg_size_in_bytes - - avg_time_in_millis - - total_operations - - total_size_in_bytes - - total_time_in_millis GeoShapeRelation: type: string enum: diff --git a/spec/schemas/indices.stats.yaml b/spec/schemas/indices.stats.yaml index d3301fcd0..eb729afb7 100644 --- a/spec/schemas/indices.stats.yaml +++ b/spec/schemas/indices.stats.yaml @@ -24,130 +24,89 @@ components: required: - description - size_in_bytes + AllIndicesStats: + type: object + properties: + primaries: + $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' + required: + - primaries + - total IndicesStats: type: object properties: + uuid: + $ref: '_common.yaml#/components/schemas/Uuid' primaries: $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' shards: type: object additionalProperties: type: array items: - $ref: '#/components/schemas/ShardStats' - total: - $ref: '#/components/schemas/IndexStats' - uuid: - $ref: '_common.yaml#/components/schemas/Uuid' - health: - $ref: '_common.yaml#/components/schemas/HealthStatus' - status: - $ref: '#/components/schemas/IndexMetadataState' - IndexStats: + $ref: '#/components/schemas/IndexShardStats' + required: + - primaries + - total + - uuid + IndexStatsBase: type: object properties: - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' docs: $ref: '_common.yaml#/components/schemas/DocStats' - fielddata: - $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' + store: + $ref: '_common.yaml#/components/schemas/StoreStats' indexing: $ref: '_common.yaml#/components/schemas/IndexingStats' - indices: - $ref: '#/components/schemas/IndicesStats' + get: + $ref: '_common.yaml#/components/schemas/GetStats' + search: + $ref: '_common.yaml#/components/schemas/SearchStats' merges: $ref: '_common.yaml#/components/schemas/MergesStats' - query_cache: - $ref: '_common.yaml#/components/schemas/QueryCacheStats' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' refresh: $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' - segments: - $ref: '_common.yaml#/components/schemas/SegmentsStats' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' - translog: - $ref: '_common.yaml#/components/schemas/TranslogStats' + flush: + $ref: '_common.yaml#/components/schemas/FlushStats' warmer: $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - ShardsTotalStats: - type: object - properties: - total_count: - type: number - required: - - total_count - ShardStats: - type: object - properties: - commit: - $ref: '#/components/schemas/ShardCommit' - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' - docs: - $ref: '_common.yaml#/components/schemas/DocStats' + query_cache: + $ref: '_common.yaml#/components/schemas/QueryCacheStats' fielddata: $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' - indexing: - $ref: '_common.yaml#/components/schemas/IndexingStats' - mappings: - $ref: '#/components/schemas/MappingStats' - merges: - $ref: '_common.yaml#/components/schemas/MergesStats' - shard_path: - $ref: '#/components/schemas/ShardPath' - query_cache: - $ref: '#/components/schemas/ShardQueryCache' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' - refresh: - $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - retention_leases: - $ref: '#/components/schemas/ShardRetentionLeases' - routing: - $ref: '#/components/schemas/ShardRouting' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' + completion: + $ref: '_common.yaml#/components/schemas/CompletionStats' segments: $ref: '_common.yaml#/components/schemas/SegmentsStats' - seq_no: - $ref: '#/components/schemas/ShardSequenceNumber' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' translog: $ref: '_common.yaml#/components/schemas/TranslogStats' - warmer: - $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shards: - type: object - additionalProperties: - type: object - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - indices: - $ref: '#/components/schemas/IndicesStats' + request_cache: + $ref: '_common.yaml#/components/schemas/RequestCacheStats' + recovery: + $ref: '_common.yaml#/components/schemas/RecoveryStats' + IndexStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + IndexShardStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + properties: + routing: + $ref: '#/components/schemas/ShardRouting' + commit: + $ref: '#/components/schemas/ShardCommit' + seq_no: + $ref: '#/components/schemas/ShardSequenceNumber' + retention_leases: + $ref: '#/components/schemas/ShardRetentionLeases' + shard_path: + $ref: '#/components/schemas/ShardPath' ShardCommit: type: object properties: @@ -166,18 +125,6 @@ components: - id - num_docs - user_data - MappingStats: - type: object - properties: - total_count: - type: number - total_estimated_overhead: - $ref: '_common.yaml#/components/schemas/ByteSize' - total_estimated_overhead_in_bytes: - type: number - required: - - total_count - - total_estimated_overhead_in_bytes ShardPath: type: object properties: @@ -191,31 +138,6 @@ components: - data_path - is_custom_data_path - state_path - ShardQueryCache: - type: object - properties: - cache_count: - type: number - cache_size: - type: number - evictions: - type: number - hit_count: - type: number - memory_size_in_bytes: - type: number - miss_count: - type: number - total_count: - type: number - required: - - cache_count - - cache_size - - evictions - - hit_count - - memory_size_in_bytes - - miss_count - - total_count ShardRetentionLeases: type: object properties: @@ -281,9 +203,4 @@ components: required: - global_checkpoint - local_checkpoint - - max_seq_no - IndexMetadataState: - type: string - enum: - - close - - open + - max_seq_no \ No newline at end of file diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 59378e88a..ff0efdc21 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -250,7 +250,7 @@ components: indexing_pressure: $ref: '#/components/schemas/IndexingPressure' indices: - $ref: 'indices.stats.yaml#/components/schemas/ShardStats' + $ref: 'indices.stats.yaml#/components/schemas/IndexShardStats' shard_indexing_pressure: $ref: '#/components/schemas/ShardIndexingPressureStats' search_backpressure: From cb320b5482551c4f28afa26ff0d1653332699722 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Mon, 12 Aug 2024 11:26:49 -0400 Subject: [PATCH 21/31] Request sort in rest_total_hits_as_int tests (#493) Signed-off-by: Andy Wick --- .../_core/search/rest_total_hits_as_int.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 3d01d28da..8e7a04351 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -11,6 +11,7 @@ prologues: director: Bennett Miller title: Moneyball year: 2011 + order: 1 status: [201] - path: /movies/_doc method: POST @@ -21,6 +22,7 @@ prologues: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 status: [201] epilogues: - path: /movies @@ -37,26 +39,26 @@ chapters: payload: query: match_all: {} + sort: order response: status: 200 payload: timed_out: false hits: total: 2 - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 - synopsis: Search with rest_total_hits_as_int=false. path: /{index}/_search parameters: @@ -67,6 +69,7 @@ chapters: payload: query: match_all: {} + sort: order response: status: 200 payload: @@ -75,20 +78,19 @@ chapters: total: value: 2 relation: eq - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 - synopsis: Search with rest_total_hits_as_int=false track_total_hits=1. path: /{index}/_search parameters: @@ -100,6 +102,7 @@ chapters: track_total_hits: 1 query: match_all: {} + sort: order response: status: 200 payload: @@ -108,18 +111,17 @@ chapters: total: value: 1 relation: gte - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 From 382883848467c7f2edef81c074446c73e88bb80f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 13:03:20 -0400 Subject: [PATCH 22/31] Catch response deserialization errors. (#478) * Catch unexpected errors. Signed-off-by: dblock * Eat the error instead of catching it. Signed-off-by: dblock * Distinguish errors and errors. Signed-off-by: dblock --------- Signed-off-by: dblock --- tools/src/tester/ChapterEvaluator.ts | 10 +++-- tools/src/tester/ChapterReader.ts | 18 ++++---- .../fixtures/evals/error/chapter_error.yaml | 1 - .../fixtures/evals/error/prologue_error.yaml | 1 - .../fixtures/stories/failed/not_found.yaml | 1 - .../tests/tester/integ/StoryEvaluator.test.ts | 44 +++++++++++++++++++ 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 0be147f4b..ae73f90a3 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -122,15 +122,17 @@ export default class ChapterEvaluator { #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation { const expected_status = chapter.response?.status ?? 200 - if (response.status === expected_status) return { result: Result.PASSED } + if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED } - const result: Evaluation = { + let result: Evaluation = { result: Result.ERROR, message: _.join(_.compact([ - `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`, + expected_status == response.status ? + `Received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.` : + `Expected status ${expected_status}, but received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.`, response.message ]), ' ') - }; + } if (response.error !== undefined) { result.error = response.error as Error diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index ae26e9920..26dd02753 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -56,16 +56,16 @@ export default class ChapterReader { }).catch(e => { if (e.response == null) { this.logger.info(`<= ERROR: ${e}`) - throw e + response.message = e.message + response.error = e + } else { + response.status = e.response.status + response.content_type = e.response.headers['content-type']?.split(';')[0] + const payload = this.#deserialize_payload(e.response.data, response.content_type) + if (payload !== undefined) response.payload = payload.error + response.message = payload.error?.reason ?? e.response.statusText + this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } - response.status = e.response.status - response.content_type = e.response.headers['content-type']?.split(';')[0] - const payload = this.#deserialize_payload(e.response.data, response.content_type) - if (payload !== undefined) response.payload = payload.error - response.message = payload.error?.reason ?? e.response.statusText - response.error = e - - this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) }) return response as ActualResponse } diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index 450d9c5a3..fab1f0cc4 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -33,7 +33,6 @@ chapters: result: ERROR message: 'Expected status 200, but received 404: application/json. no such index [undefined]' - error: Request failed with status code 404 payload_body: result: SKIPPED payload_schema: diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml index 5ff63bb87..d91beb82f 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml @@ -16,7 +16,6 @@ prologues: overall: result: ERROR message: no such index [does_not_exists] - error: Request failed with status code 404 chapters: - title: This chapter be skipped. diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/failed/not_found.yaml index 0b61b03a9..2bedb9c68 100644 --- a/tools/tests/tester/fixtures/stories/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/stories/failed/not_found.yaml @@ -29,4 +29,3 @@ chapters: index: movies response: status: 404 - diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index 7abc9dbfd..ac5fc60ef 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -7,6 +7,7 @@ * compatible open source license. */ +import { Result } from 'tester/types/eval.types' import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from '../helpers' const { story_evaluator, opensearch_http_client } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') @@ -16,6 +17,10 @@ beforeAll(async () => { expect(info.version).toBeDefined() }) +afterEach(() => { + jest.resetAllMocks() +}) + test('passed', async () => { const actual = await load_actual_evaluation(story_evaluator, 'passed') const expected = load_expected_evaluation('passed') @@ -57,3 +62,42 @@ test('skipped/semver', async () => { const expected = load_expected_evaluation('skipped/semver') expect(actual).toEqual(expected) }) + +test('with an unexpected error deserializing data', async () => { + opensearch_http_client.request = jest.fn().mockRejectedValue(new Error('This was unexpected.')) + const actual = await load_actual_evaluation(story_evaluator, 'passed') + expect(actual.result).toEqual(Result.ERROR) + expect(actual.chapters && actual.chapters[0]).toEqual({ + title: "This PUT /{index} chapter should pass.", + path: 'PUT /{index}', + overall: { + result: Result.ERROR + }, + request: { + parameters: { + index: { + result: Result.PASSED + }, + }, + request: { + result: Result.PASSED + } + }, + response: { + output_values: { + result: Result.SKIPPED + }, + payload_body: { + result: Result.SKIPPED + }, + payload_schema: { + result: Result.SKIPPED + }, + status: { + error: 'This was unexpected.', + message: 'Expected status 200, but received none: unknown. This was unexpected.', + result: Result.ERROR + } + } + }) +}) \ No newline at end of file From 9eb37325f5ec46dd2c204d0c933ddfc413056e4a Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Mon, 12 Aug 2024 14:10:40 -0400 Subject: [PATCH 23/31] Adds test for APIs related to security plugin and updates spec to add new APIs. (#439) * Updates the missing item types for security API related schemas Signed-off-by: Darshit Chanpura * Adds tests for some APIs Signed-off-by: Darshit Chanpura * Fixes schema path and lint error in a file Signed-off-by: Darshit Chanpura * Adds version check for account API test Signed-off-by: Darshit Chanpura * Adds a CHANGELOG entry Signed-off-by: Darshit Chanpura * Fixes tests to expect status as integer Signed-off-by: Darshit Chanpura * Adds base tests for all APIs Signed-off-by: Darshit Chanpura * Fixes linter errors and complete payloads and request bodies Signed-off-by: Darshit Chanpura * Adds API spec for the new certificates API and adds tests Signed-off-by: Darshit Chanpura * Adds missing new lines at the end of files and removes any extra lines added by linter Signed-off-by: Darshit Chanpura * Fixes lint errors Signed-off-by: Darshit Chanpura * Fixes test spec lint Signed-off-by: Darshit Chanpura * Fixes without api prefix tests Signed-off-by: Darshit Chanpura * Fixes accounts tests Signed-off-by: Darshit Chanpura * Fixes action-groups tests Signed-off-by: Darshit Chanpura * Fixes account, allowlist and audit tests Signed-off-by: Darshit Chanpura * Fixes certificates tests Signed-off-by: Darshit Chanpura * Fixes internal users test Signed-off-by: Darshit Chanpura * Fixes nodesdn test Signed-off-by: Darshit Chanpura * Fixes roles test Signed-off-by: Darshit Chanpura * Fixes rolesmapping test Signed-off-by: Darshit Chanpura * Fixes securityconfig test Signed-off-by: Darshit Chanpura * Fixes ssl_certs tests Signed-off-by: Darshit Chanpura * Fixes tenancyconfig tests Signed-off-by: Darshit Chanpura * Fixes tenants tests Signed-off-by: Darshit Chanpura * Fixes tokens and upgrade tests Signed-off-by: Darshit Chanpura * Fixes user tests Signed-off-by: Darshit Chanpura * Fixes validate tests and run linter Signed-off-by: Darshit Chanpura * Fixes spec Signed-off-by: Darshit Chanpura * Adds missing new lines to EOFs Signed-off-by: Darshit Chanpura * Address changes in CHANGELOG Signed-off-by: Darshit Chanpura * Updates parameter descriptions Signed-off-by: Darshit Chanpura * Re-organize folder structure Signed-off-by: Darshit Chanpura * Moved security tests inside plugins folder Signed-off-by: Darshit Chanpura * Updates test spec workflow to run security tests Signed-off-by: Darshit Chanpura * Adds security test specific docker compose file Signed-off-by: Darshit Chanpura * Cleans commented code and updates test-spec to run for 2.16 only Signed-off-by: Darshit Chanpura * Refactor s boolean to be true boolean Signed-off-by: Darshit Chanpura * Move security tests to default folder and updates spec file Signed-off-by: Darshit Chanpura * Fixes 1.3 tests Signed-off-by: Darshit Chanpura * Fixes 2.0 test failures Signed-off-by: Darshit Chanpura * Fixes TLS cipher versiion payload issue Signed-off-by: Darshit Chanpura * Re-verifies auth token fix Signed-off-by: Darshit Chanpura * Checks for newer images and pulls them before running tests Signed-off-by: Darshit Chanpura * Updates the docker compose command Signed-off-by: Darshit Chanpura * Removes sha ref for staging branches Signed-off-by: Darshit Chanpura * Updates shas Signed-off-by: Darshit Chanpura * Renames everything to match the API path Signed-off-by: Darshit Chanpura --------- Signed-off-by: Darshit Chanpura --- .cspell | 1 + .github/workflows/test-spec.yml | 4 +- CHANGELOG.md | 3 + spec/namespaces/security.yaml | 417 ++++++++++++++---- spec/schemas/security._common.yaml | 214 ++++++--- .../default/security/api/_upgrade_check.yaml | 11 + .../security/api/_upgrade_perform.yaml | 14 + tests/default/security/api/account.yaml | 21 + tests/default/security/api/actiongroups.yaml | 59 +++ tests/default/security/api/allowlist.yaml | 31 ++ tests/default/security/api/audit.yaml | 20 + tests/default/security/api/audit/config.yaml | 39 ++ tests/default/security/api/authtoken.yaml | 10 + tests/default/security/api/cache.yaml | 25 ++ tests/default/security/api/certificates.yaml | 35 ++ .../security/api/generateonbehalfoftoken.yaml | 18 + tests/default/security/api/internalusers.yaml | 60 +++ .../api/internalusers.yml/authtoken.yml | 35 ++ tests/default/security/api/migrate.yaml | 11 + tests/default/security/api/nodesdn.yaml | 61 +++ .../default/security/api/permissionsinfo.yaml | 10 + tests/default/security/api/roles.yaml | 75 ++++ tests/default/security/api/rolesmapping.yaml | 77 ++++ .../default/security/api/securityconfig.yaml | 22 + .../security/api/securityconfig/config.yaml | 43 ++ tests/default/security/api/ssl/certs.yml | 12 + .../security/api/ssl/http/reloadcerts.yaml | 12 + .../api/ssl/transport/reloadcerts.yaml | 12 + .../default/security/api/tenancy/config.yaml | 22 + tests/default/security/api/tenants.yaml | 56 +++ tests/default/security/api/user.yaml | 37 ++ .../default/security/api/user/authtoken.yaml | 35 ++ tests/default/security/api/validate.yaml | 13 + tests/default/security/authinfo.yaml | 34 ++ tests/default/security/dashboardsinfo.yaml | 15 + tests/default/security/health.yaml | 27 ++ tests/default/security/sslinfo.yaml | 25 ++ tests/default/security/tenantinfo.yaml | 17 + tests/default/security/whoami.yaml | 24 + tests/default/security/whoamiprotected.yaml | 15 + 40 files changed, 1503 insertions(+), 169 deletions(-) create mode 100644 tests/default/security/api/_upgrade_check.yaml create mode 100644 tests/default/security/api/_upgrade_perform.yaml create mode 100644 tests/default/security/api/account.yaml create mode 100644 tests/default/security/api/actiongroups.yaml create mode 100644 tests/default/security/api/allowlist.yaml create mode 100644 tests/default/security/api/audit.yaml create mode 100644 tests/default/security/api/audit/config.yaml create mode 100644 tests/default/security/api/authtoken.yaml create mode 100644 tests/default/security/api/cache.yaml create mode 100644 tests/default/security/api/certificates.yaml create mode 100644 tests/default/security/api/generateonbehalfoftoken.yaml create mode 100644 tests/default/security/api/internalusers.yaml create mode 100644 tests/default/security/api/internalusers.yml/authtoken.yml create mode 100644 tests/default/security/api/migrate.yaml create mode 100644 tests/default/security/api/nodesdn.yaml create mode 100644 tests/default/security/api/permissionsinfo.yaml create mode 100644 tests/default/security/api/roles.yaml create mode 100644 tests/default/security/api/rolesmapping.yaml create mode 100644 tests/default/security/api/securityconfig.yaml create mode 100644 tests/default/security/api/securityconfig/config.yaml create mode 100644 tests/default/security/api/ssl/certs.yml create mode 100644 tests/default/security/api/ssl/http/reloadcerts.yaml create mode 100644 tests/default/security/api/ssl/transport/reloadcerts.yaml create mode 100644 tests/default/security/api/tenancy/config.yaml create mode 100644 tests/default/security/api/tenants.yaml create mode 100644 tests/default/security/api/user.yaml create mode 100644 tests/default/security/api/user/authtoken.yaml create mode 100644 tests/default/security/api/validate.yaml create mode 100644 tests/default/security/authinfo.yaml create mode 100644 tests/default/security/dashboardsinfo.yaml create mode 100644 tests/default/security/health.yaml create mode 100644 tests/default/security/sslinfo.yaml create mode 100644 tests/default/security/tenantinfo.yaml create mode 100644 tests/default/security/whoami.yaml create mode 100644 tests/default/security/whoamiprotected.yaml diff --git a/.cspell b/.cspell index 900b6c40a..1fcd3199c 100644 --- a/.cspell +++ b/.cspell @@ -165,6 +165,7 @@ subqueries subschemas subword syserr +tcnative tdigest tenantinfo termvectors diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index d9ed81aea..13a218e29 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -34,10 +34,10 @@ jobs: tests: snapshot - version: 2.17.0 hub: opensearchstaging - ref: '@sha256:6398c27d7560626ed6b0ba28b3d6b20b7f00c6d94abf45ad3a820f8eeb3d61a3' + ref: '@sha256:ed4274522a50228f41b50f1a7ea86e6b52fa6737072fc151b2624d22aff80d56' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:101681eea630393f8caf5987dd023a975a9656b63090a07bfdfe6ad2f73f0640' + ref: '@sha256:cab6f71b284485c44306f8f4849ad520283c2a32ece617109b38183ba29cc401' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index a5821316f..25a085e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `observability` namespace API specifications ([#474](https://github.com/opensearch-project/opensearch-api-specification/pull/474)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) +- Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) ### Changed @@ -111,6 +113,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) +- Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) ### Security diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index f41647f59..72da8905e 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -60,8 +60,6 @@ paths: x-operation-group: security.post_dashboards_info x-version-added: '1.0' description: Updates the current security-dashboards plugin configuration. - requestBody: - $ref: '#/components/requestBodies/security.post_dashboards_info' responses: '200': $ref: '#/components/responses/security.post_dashboards_info@200' @@ -101,6 +99,8 @@ paths: responses: '200': $ref: '#/components/responses/security.tenant_info@200' + '403': + $ref: '#/components/responses/security.tenant_info@403' '500': $ref: '#/components/responses/security.tenant_info@500' post: @@ -111,13 +111,15 @@ paths: responses: '200': $ref: '#/components/responses/security.tenant_info@200' + '403': + $ref: '#/components/responses/security.tenant_info@403' '500': $ref: '#/components/responses/security.tenant_info@500' /_plugins/_security/whoami: get: operationId: security.who_am_i.0 x-operation-group: security.who_am_i - x-version-added: '1.0' + x-version-added: '2.0' description: Gets the user identity related information for currently logged in user. responses: '200': @@ -127,7 +129,7 @@ paths: post: operationId: security.who_am_i.1 x-operation-group: security.who_am_i - x-version-added: '1.0' + x-version-added: '2.0' description: Gets the user identity related information for currently logged in user. responses: '200': @@ -145,7 +147,7 @@ paths: $ref: '#/components/responses/security.who_am_i_protected@200' '500': $ref: '#/components/responses/security.who_am_i_protected@500' - /_plugins/_security/_upgrade_check: + /_plugins/_security/api/_upgrade_check: get: operationId: security.config_upgrade_check.0 x-operation-group: security.config_upgrade_check @@ -156,7 +158,7 @@ paths: responses: '200': $ref: '#/components/responses/security.config_upgrade_check@200' - /_plugins/_security/_upgrade_perform: + /_plugins/_security/api/_upgrade_perform: post: operationId: security.config_upgrade_perform.0 x-operation-group: security.config_upgrade_perform @@ -169,6 +171,8 @@ paths: responses: '200': $ref: '#/components/responses/security.config_upgrade_perform@200' + '400': + $ref: '#/components/responses/security.config_upgrade_perform@400' /_plugins/_security/api/account: get: operationId: security.get_account_details.0 @@ -192,6 +196,8 @@ paths: responses: '200': $ref: '#/components/responses/security.change_password@200' + '403': + $ref: '#/components/responses/security.change_password@403' /_plugins/_security/api/actiongroups: get: operationId: security.get_action_groups.0 @@ -242,6 +248,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_action_group@200' + '201': + $ref: '#/components/responses/security.create_action_group@201' patch: operationId: security.patch_action_group.0 x-operation-group: security.patch_action_group @@ -272,17 +280,19 @@ paths: get: operationId: security.get_allowlist.0 x-operation-group: security.get_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Retrieves the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api responses: '200': $ref: '#/components/responses/security.get_allowlist@200' + '403': + $ref: '#/components/responses/security.get_allowlist@403' put: operationId: security.create_allowlist.0 x-operation-group: security.create_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Creates or replaces the allowlisted APIs. Accessible via Super Admin certificate or REST API permission. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -291,10 +301,12 @@ paths: responses: '200': $ref: '#/components/responses/security.create_allowlist@200' + '403': + $ref: '#/components/responses/security.create_allowlist@403' patch: operationId: security.patch_allowlist.0 x-operation-group: security.patch_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Updates the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -303,6 +315,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_allowlist@200' + '403': + $ref: '#/components/responses/security.patch_allowlist@403' /_plugins/_security/api/audit: get: operationId: security.get_audit_configuration.0 @@ -386,6 +400,39 @@ paths: responses: '200': $ref: '#/components/responses/security.flush_cache@200' + /_plugins/_security/api/certificates: + get: + operationId: security.get_all_certificates.0 + x-operation-group: security.get_all_certificates + x-version-added: '2.15' + description: Retrieves the cluster security certificates. + parameters: + - $ref: '#/components/parameters/security.get_all_certificates::query.cert_type' + - $ref: '#/components/parameters/security.get_all_certificates::query.timeout' + responses: + '200': + $ref: '#/components/responses/security.get_all_certificates@200' + '403': + $ref: '#/components/responses/security.get_all_certificates@403' + '500': + $ref: '#/components/responses/security.get_all_certificates@500' + /_plugins/_security/api/certificates/{node_id}: + get: + operationId: security.get_node_certificates.0 + x-operation-group: security.get_node_certificates + x-version-added: '2.15' + description: Retrieves the given node's security certificates. + parameters: + - $ref: '#/components/parameters/security.get_node_certificates::path.node_id' + - $ref: '#/components/parameters/security.get_node_certificates::query.cert_type' + - $ref: '#/components/parameters/security.get_node_certificates::query.timeout' + responses: + '200': + $ref: '#/components/responses/security.get_node_certificates@200' + '403': + $ref: '#/components/responses/security.get_node_certificates@403' + '500': + $ref: '#/components/responses/security.get_node_certificates@500' /_plugins/_security/api/generateonbehalfoftoken: post: operationId: security.generate_obo_token.0 @@ -481,7 +528,7 @@ paths: post: operationId: security.generate_user_token.0 x-operation-group: security.generate_user_token - x-version-added: '1.0' + x-version-added: '2.7' description: Generates authorization token for the given user. parameters: - $ref: '#/components/parameters/security.generate_user_token::path.username' @@ -514,6 +561,8 @@ paths: responses: '200': $ref: '#/components/responses/security.get_distinguished_names@200' + '400': + $ref: '#/components/responses/security.get_distinguished_names@400' '403': $ref: '#/components/responses/security.get_distinguished_names@403' patch: @@ -528,6 +577,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_distinguished_names@200' + '400': + $ref: '#/components/responses/security.patch_distinguished_names@400' '403': $ref: '#/components/responses/security.patch_distinguished_names@403' /_plugins/_security/api/nodesdn/{cluster_name}: @@ -544,6 +595,8 @@ paths: responses: '200': $ref: '#/components/responses/security.get_distinguished_name@200' + '400': + $ref: '#/components/responses/security.get_distinguished_name@400' '403': $ref: '#/components/responses/security.get_distinguished_name@403' put: @@ -560,6 +613,8 @@ paths: responses: '200': $ref: '#/components/responses/security.update_distinguished_name@200' + '400': + $ref: '#/components/responses/security.update_distinguished_name@400' '403': $ref: '#/components/responses/security.update_distinguished_name@403' patch: @@ -574,6 +629,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_distinguished_name@200' + '400': + $ref: '#/components/responses/security.patch_distinguished_name@400' '403': $ref: '#/components/responses/security.patch_distinguished_name@403' delete: @@ -588,6 +645,8 @@ paths: responses: '200': $ref: '#/components/responses/security.delete_distinguished_name@200' + '400': + $ref: '#/components/responses/security.delete_distinguished_name@400' '403': $ref: '#/components/responses/security.delete_distinguished_name@403' /_plugins/_security/api/permissionsinfo: @@ -653,6 +712,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_role@200' + '201': + $ref: '#/components/responses/security.create_role@201' patch: operationId: security.patch_role.0 x-operation-group: security.patch_role @@ -733,6 +794,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_role_mapping@200' + '201': + $ref: '#/components/responses/security.create_role_mapping@201' patch: operationId: security.patch_role_mapping.0 x-operation-group: security.patch_role_mapping @@ -765,7 +828,7 @@ paths: get: operationId: security.get_configuration.0 x-operation-group: security.get_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: Returns the current Security plugin configuration in JSON format. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-configuration @@ -775,7 +838,7 @@ paths: patch: operationId: security.patch_configuration.0 x-operation-group: security.patch_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: A PATCH call is used to update the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-configuration @@ -784,11 +847,13 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_configuration@200' + '403': + $ref: '#/components/responses/security.patch_configuration@403' /_plugins/_security/api/securityconfig/config: put: operationId: security.update_configuration.0 x-operation-group: security.update_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: Adds or updates the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#update-configuration @@ -797,11 +862,13 @@ paths: responses: '200': $ref: '#/components/responses/security.update_configuration@200' + '403': + $ref: '#/components/responses/security.update_configuration@403' /_plugins/_security/api/ssl/certs: get: operationId: security.get_certificates.0 x-operation-group: security.get_certificates - x-version-added: '1.0' + x-version-added: '2.0' description: Retrieves the cluster security certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-certificates @@ -810,11 +877,13 @@ paths: $ref: '#/components/responses/security.get_certificates@200' '400': $ref: '#/components/responses/security.get_certificates@400' + '403': + $ref: '#/components/responses/security.get_certificates@403' /_plugins/_security/api/ssl/http/reloadcerts: put: operationId: security.reload_http_certificates.0 x-operation-group: security.reload_http_certificates - x-version-added: '1.0' + x-version-added: '2.8' description: Reload HTTP layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-http-certificates @@ -823,11 +892,13 @@ paths: $ref: '#/components/responses/security.reload_http_certificates@200' '400': $ref: '#/components/responses/security.reload_http_certificates@400' + '403': + $ref: '#/components/responses/security.reload_http_certificates@403' /_plugins/_security/api/ssl/transport/reloadcerts: put: operationId: security.reload_transport_certificates.0 x-operation-group: security.reload_transport_certificates - x-version-added: '1.0' + x-version-added: '2.8' description: Reload Transport layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-transport-certificates @@ -836,6 +907,8 @@ paths: $ref: '#/components/responses/security.reload_transport_certificates@200' '400': $ref: '#/components/responses/security.reload_transport_certificates@400' + '403': + $ref: '#/components/responses/security.reload_transport_certificates@403' /_plugins/_security/api/tenancy/config: get: operationId: security.get_tenancy_config.0 @@ -917,6 +990,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_tenant@200' + '201': + $ref: '#/components/responses/security.create_tenant@201' '400': $ref: '#/components/responses/security.create_tenant@400' patch: @@ -981,6 +1056,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_user_legacy@200' + '201': + $ref: '#/components/responses/security.create_user_legacy@201' delete: operationId: security.delete_user_legacy.0 x-operation-group: security.delete_user_legacy @@ -995,15 +1072,13 @@ paths: post: operationId: security.generate_user_token_legacy.0 x-operation-group: security.generate_user_token_legacy - x-version-added: '1.0' - description: Generates authorization token for the given user. Legacy API. + x-version-added: '2.7' + description: Generates authorization token for the given user. Legacy API. Not Implemented. parameters: - $ref: '#/components/parameters/security.generate_user_token_legacy::path.username' responses: - '200': - $ref: '#/components/responses/security.generate_user_token_legacy@200' - '400': - $ref: '#/components/responses/security.generate_user_token_legacy@400' + '501': + $ref: '#/components/responses/security.generate_user_token_legacy@501' /_plugins/_security/api/validate: get: operationId: security.validate.0 @@ -1064,9 +1139,7 @@ components: content: application/json: schema: - type: array - items: - $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' required: true security.create_user: content: @@ -1203,12 +1276,6 @@ components: items: $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' required: true - security.post_dashboards_info: - content: - application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/DashboardsInfo' - required: false security.update_audit_configuration: content: application/json: @@ -1225,7 +1292,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' + $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' responses: security.authinfo@200: content: @@ -1245,13 +1312,16 @@ components: security.cache@501: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/MethodNotImplemented' + schema: null security.change_password@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.change_password@403: + content: + application/json: + schema: null security.config_upgrade_check@200: content: application/json: @@ -1262,36 +1332,63 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UpgradePerform' + security.config_upgrade_perform@400: + content: + application/json: + schema: null security.create_action_group@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_action_group@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_allowlist@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.create_allowlist@403: + content: + application/json: + schema: null security.create_role@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_role@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_role_mapping@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_role_mapping@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_tenant@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' - security.create_tenant@400: + security.create_tenant@201: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + $ref: '../schemas/security._common.yaml#/components/schemas/Created' + security.create_tenant@400: + content: + application/json: + schema: null security.create_update_tenancy_config@200: content: application/json: @@ -1300,8 +1397,7 @@ components: security.create_update_tenancy_config@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.create_user@200: content: application/json: @@ -1312,6 +1408,11 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_user_legacy@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.delete_action_group@200: content: application/json: @@ -1322,11 +1423,14 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.delete_distinguished_name@400: + content: + application/json: + schema: null security.delete_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.delete_role@200: content: application/json: @@ -1345,8 +1449,7 @@ components: security.delete_tenant@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.delete_user@200: content: application/json: @@ -1369,9 +1472,10 @@ components: $ref: '../schemas/security._common.yaml#/components/schemas/GenerateOBOToken' security.generate_obo_token@400: content: + text/plain: + type: string application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.generate_user_token@200: content: application/json: @@ -1380,18 +1484,11 @@ components: security.generate_user_token@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' - security.generate_user_token_legacy@200: - content: - application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Ok' - security.generate_user_token_legacy@400: + schema: null + security.generate_user_token_legacy@501: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_account_details@200: content: application/json: @@ -1412,6 +1509,10 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.get_allowlist@403: + content: + application/json: + schema: null security.get_audit_configuration@200: content: application/json: @@ -1422,16 +1523,47 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificates' + security.get_all_certificates@200: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificatesNew' + security.get_all_certificates@403: + content: + application/json: + schema: null + security.get_all_certificates@500: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' + security.get_node_certificates@200: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificatesNew' + security.get_node_certificates@403: + content: + application/json: + schema: null security.get_certificates@400: + content: + application/json: + schema: null + security.get_certificates@403: + content: + application/json: + schema: null + security.get_node_certificates@500: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.get_configuration@200: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/DynamicConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/SecurityConfig' security.get_dashboards_info@200: content: application/json: @@ -1448,21 +1580,28 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' + security.get_distinguished_name@400: + content: + application/json: + schema: null security.get_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.get_distinguished_names@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNamesMap' + security.get_distinguished_names@400: + description: Show nodesDn setting for given cluster. + content: + application/json: + schema: null security.get_distinguished_names@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.get_permissions_info@200: content: application/json: @@ -1516,8 +1655,7 @@ components: security.get_tenancy_config@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_tenants@200: content: application/json: @@ -1526,8 +1664,7 @@ components: security.get_tenants@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_user@200: content: application/json: @@ -1561,8 +1698,7 @@ components: security.migrate@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_action_group@200: content: application/json: @@ -1578,6 +1714,10 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.patch_allowlist@403: + content: + application/json: + schema: null security.patch_audit_configuration@200: content: application/json: @@ -1588,26 +1728,36 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_configuration@403: + content: + application/json: + schema: null security.patch_distinguished_name@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_distinguished_name@400: + content: + application/json: + schema: null security.patch_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.patch_distinguished_names@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_distinguished_names@400: + content: + application/json: + schema: null security.patch_distinguished_names@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.patch_role@200: content: application/json: @@ -1616,8 +1766,7 @@ components: security.patch_role@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_role_mapping@200: content: application/json: @@ -1626,8 +1775,7 @@ components: security.patch_role_mapping@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_role_mappings@200: content: application/json: @@ -1636,8 +1784,7 @@ components: security.patch_role_mappings@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_roles@200: content: application/json: @@ -1646,8 +1793,7 @@ components: security.patch_roles@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_tenant@200: content: application/json: @@ -1656,8 +1802,7 @@ components: security.patch_tenant@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_tenants@200: content: application/json: @@ -1666,8 +1811,7 @@ components: security.patch_tenants@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_user@200: content: application/json: @@ -1696,8 +1840,11 @@ components: security.reload_http_certificates@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null + security.reload_http_certificates@403: + content: + application/json: + schema: null security.reload_transport_certificates@200: content: application/json: @@ -1706,13 +1853,22 @@ components: security.reload_transport_certificates@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null + security.reload_transport_certificates@403: + content: + application/json: + schema: null security.tenant_info@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/TenantInfo' + security.tenant_info@403: + content: + text/plain: + type: string + application/json: + schema: null security.tenant_info@500: content: application/json: @@ -1728,16 +1884,23 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.update_configuration@403: + content: + application/json: + schema: null security.update_distinguished_name@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.update_distinguished_name@400: + content: + application/json: + schema: null security.update_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.validate@200: content: application/json: @@ -1746,8 +1909,7 @@ components: security.validate@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.who_am_i@200: content: application/json: @@ -1789,117 +1951,165 @@ components: description: The name of the action group to create or replace. schema: type: string - description: The name of the action group to create or replace. required: true security.create_role::path.role: name: role in: path + description: The name of the role to be created. schema: type: string required: true security.create_role_mapping::path.role: name: role in: path + description: The name of the role to create a role mapping for. schema: type: string required: true security.create_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to be created. schema: type: string required: true security.create_user::path.username: name: username in: path + description: The name of the user to be created. schema: type: string required: true security.create_user_legacy::path.username: name: username in: path + description: The name of the user to be created. schema: type: string required: true security.delete_action_group::path.action_group: name: action_group in: path - description: Action group to delete. + description: The name of the action group to delete. schema: type: string - description: Action group to delete. required: true security.delete_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to delete from list of distinguished names. schema: type: string required: true security.delete_role::path.role: name: role in: path + description: The name of the role to delete. schema: type: string required: true security.delete_role_mapping::path.role: name: role in: path + description: The name of the role whose mapping needs to delete. schema: type: string required: true security.delete_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to delete. schema: type: string required: true security.delete_user::path.username: name: username in: path + description: The name of the user to delete. schema: type: string required: true security.delete_user_legacy::path.username: name: username in: path + description: The name of the user to delete. schema: type: string required: true security.generate_user_token::path.username: name: username in: path + description: The name of the user for whom an auth token is to be vended. schema: type: string required: true security.generate_user_token_legacy::path.username: name: username in: path + description: The name of the user for whom an auth token is to be vended. schema: type: string required: true security.get_action_group::path.action_group: name: action_group in: path - description: Action group to retrieve. + description: The name of the action group to retrieve. schema: type: string - description: Action group to retrieve. required: true + security.get_node_certificates::path.node_id: + name: node_id + in: path + description: The full-id of the node to retrieve certificates. + schema: + type: string + required: true + security.get_all_certificates::query.cert_type: + name: cert_type + in: query + description: The type of certificates (HTTP, TRANSPORT, ALL) to retrieve from all nodes. + schema: + type: string + required: false + security.get_node_certificates::query.cert_type: + name: cert_type + in: query + description: The type of certificates (HTTP, TRANSPORT, ALL) to retrieve for a node. + schema: + type: string + required: false + security.get_all_certificates::query.timeout: + name: timeout + in: query + description: The maximum duration, in seconds, to be spent to retrieve certificates from all nodes. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + required: false + security.get_node_certificates::query.timeout: + name: timeout + in: query + description: The maximum duration, in seconds, to be spent to retrieve a node's certificates. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + required: false security.get_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to retrieve nodes DN setting for. schema: type: string required: true security.get_distinguished_name::query.show_all: name: show_all in: query + description: A boolean flag to include/exclude static nodes DN from final result. schema: type: boolean required: false security.get_distinguished_names::query.show_all: name: show_all in: query + description: A boolean flag to include/exclude static nodes DN from final result. schema: type: boolean required: false @@ -1918,80 +2128,91 @@ components: security.get_sslinfo::query.show_dn: name: show_dn in: query - description: The domain names from all certificates. + description: A boolean flag to indicate whether all domain names should be returned. schema: - type: string - description: A boolean flag to indicate whether all domain names should be returned. + type: [boolean, string] required: false security.get_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to retrieve. schema: type: string required: true security.get_user::path.username: name: username in: path + description: The name of the user to retrieve. schema: type: string required: true security.get_user_legacy::path.username: name: username in: path + description: The name of the user to retrieve. schema: type: string required: true security.health::query.mode: name: mode in: query + description: A flag to indicate whether service should consider security-plugin's status before returning health response. `strict` mode indicates service should check security plugin status. schema: type: string required: false security.patch_action_group::path.action_group: name: action_group in: path + description: The name of the action group to update. schema: type: string required: true security.patch_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to update nodesDn value. schema: type: string required: true security.patch_role::path.role: name: role in: path + description: The name of the role to update. schema: type: string required: true security.patch_role_mapping::path.role: name: role in: path + description: The name of the role to update role-mapping for. schema: type: string required: true security.patch_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to update. schema: type: string required: true security.patch_user::path.username: name: username in: path + description: The name of the user to update. schema: type: string required: true security.update_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to create/update nodesDn value for. schema: type: string required: true security.validate::query.accept_invalid: name: accept_invalid in: query + description: A boolean flag to indicate whether invalid v6 configuration should be allowed. schema: type: boolean required: false diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index 2fcce2ec2..36c976cb0 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -18,7 +18,7 @@ components: is_internal_user: type: boolean user_requested_tenant: - type: string + type: ['null', string] backend_roles: type: array items: @@ -58,12 +58,6 @@ components: $ref: '#/components/schemas/ActionGroup' AllowListConfig: - type: object - properties: - config: - $ref: '#/components/schemas/AllowConfig' - - AllowConfig: type: object properties: enabled: @@ -103,6 +97,14 @@ components: type: array items: type: string + ignore_headers: + type: array + items: + type: string + ignore_url_params: + type: array + items: + type: string disabled_rest_categories: type: array items: @@ -134,31 +136,37 @@ components: type: string description: User's name. user_requested_tenant: - type: string + type: ['null', string] description: Name of the tenant the user wants to switch to. remote_address: - type: string + type: ['null', string] description: The IP address of remote user. backend_roles: type: array description: Backend roles associated with the user. + items: + type: string custom_attribute_names: type: array description: Name of the attributes associated with the user. + items: + type: string roles: type: array description: Roles associated with the user. + items: + type: string tenants: type: object description: Tenants the user has access to with read-write or read-only access indicator. principal: - type: string + type: ['null', string] description: User principal. peer_certificates: - type: number + type: [number, string] description: Number of peer certificates. sso_logout_url: - type: string + type: ['null', string] description: Logout url. size_of_user: type: string @@ -170,16 +178,18 @@ components: type: string description: Size of backend roles in bytes. - BadRequest: + CertificateCountPerNode: type: object properties: - status: - type: string - enum: - - 400 - message: - type: string - description: Message returned as part of BAD_REQUEST response. + total: + type: number + description: Total number of nodes. + successful: + type: number + description: Number of nodes for which certificates could be fetched. + failed: + type: number + description: Number of nodes for which certificates could not be fetched. CertificatesDetail: type: object @@ -194,7 +204,34 @@ components: type: string not_after: type: string + + CertificatesPerNode: + type: object + properties: + name: + type: string + description: Name of the node. + certificates: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificateTypes' + CertificateTypes: + type: object + properties: + http: + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesDetail' + transport: + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesDetail' + GetCertificates: type: object properties: @@ -206,6 +243,22 @@ components: type: array items: $ref: '#/components/schemas/CertificatesDetail' + + GetCertificatesNew: + type: object + properties: + _nodes: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificateCountPerNode' + cluster_name: + type: string + description: Name of this cluster. + nodes: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesPerNode' + ChangePasswordRequestContent: type: object @@ -255,6 +308,8 @@ components: config: type: array description: List of configs to be upgraded. + items: + type: string CreateTenantParams: type: object @@ -292,6 +347,8 @@ components: sign_in_options: type: array description: List of available sign-in options available. + items: + type: string password_validation_error_message: type: string description: Error message when password validation fails. @@ -321,27 +378,36 @@ components: DynamicOptions: type: object properties: - filteredAliasMode: + filtered_alias_mode: type: string - disableRestAuth: + disable_rest_auth: type: boolean - disableIntertransportAuth: + disable_intertransport_auth: type: boolean - respectRequestIndicesOptions: + respect_request_indices_options: type: boolean - kibana: {} - http: {} - authc: {} - authz: {} - authFailureListeners: {} - doNotFailOnForbidden: + opensearch-dashboards: + type: object + kibana: + type: object + http: + type: object + authc: + type: object + authz: + type: object + auth_failure_listeners: + type: object + do_not_fail_on_forbidden: type: boolean - multiRolespanEnabled: + multi_rolespan_enabled: type: boolean - hostsResolverMode: + hosts_resolver_mode: type: string - doNotFailOnForbiddenEmpty: + do_not_fail_on_forbidden_empty: type: boolean + on_behalf_of: + type: object GenerateOBOToken: type: object @@ -360,7 +426,7 @@ components: type: object properties: message: - type: string + type: ['null', string] mode: type: string status: @@ -395,17 +461,6 @@ components: type: string description: Error message during request execution. - MethodNotImplemented: - type: object - properties: - status: - type: string - enum: - - 501 - message: - type: string - description: Message returned as part of NOT_IMPLEMENTED response. - MultiTenancyConfig: type: object properties: @@ -419,7 +474,6 @@ components: type: array items: type: string - description: Value in seconds. OBOToken: type: object @@ -440,12 +494,19 @@ components: type: object properties: status: - type: string - enum: - - 200 + type: [number, string] message: type: string description: Message returned as part of OK response. + + Created: + type: object + properties: + status: + type: [number, string] + message: + type: string + description: Message returned as part of CREATED response. PatchOperation: type: object @@ -534,22 +595,32 @@ components: type: object additionalProperties: $ref: '#/components/schemas/Role' + + SecurityConfig: + type: object + properties: + config: + $ref: '#/components/schemas/DynamicConfig' SSLInfo: type: object properties: principal: - type: string + type: ['null', string] description: User principal. peer_certificates: - type: number + type: [number, string] description: Number of certificates. peer_certificates_list: - type: array + type: [array,'null'] description: List of domain names from peer certificates. + items: + type: string local_certificates_list: type: array description: List of domain names from local certificates. + items: + type: string ssl_protocol: type: string description: Protocol for this ssl setup. @@ -560,13 +631,13 @@ components: type: boolean description: A boolean to indicate if OpenSSL is available. ssl_openssl_version: - type: string + type: [number, string] description: Version of openssl. ssl_openssl_version_string: - type: string + type: ['null', string] description: Full version string for openssl version. ssl_openssl_non_available_cause: - type: string + type: ['null', string] description: Reason for openssl unavailability. ssl_openssl_supports_key_manager_factory: type: boolean @@ -583,6 +654,20 @@ components: ssl_provider_transport_client: type: string description: Returns transport client's name. + required: + - peer_certificates + - principal + - ssl_cipher + - ssl_openssl_available + - ssl_openssl_non_available_cause + - ssl_openssl_supports_hostname_validation + - ssl_openssl_supports_key_manager_factory + - ssl_openssl_version + - ssl_openssl_version_string + - ssl_protocol + - ssl_provider_http + - ssl_provider_transport_client + - ssl_provider_transport_server Tenant: type: object @@ -613,17 +698,6 @@ components: additionalProperties: $ref: '#/components/schemas/Tenant' - Unauthorized: - type: object - properties: - status: - type: string - enum: - - 403 - message: - type: string - description: Message returned as part of FORBIDDEN response. - UpgradeCheck: type: object properties: @@ -645,6 +719,8 @@ components: User: type: object properties: + password: + type: string hash: type: string reserved: @@ -690,11 +766,11 @@ components: type: object properties: dn: - type: string + type: ['null', string] is_admin: - type: string + type: boolean is_node_certificate_request: - type: string + type: boolean TenantInfo: type: object diff --git a/tests/default/security/api/_upgrade_check.yaml b/tests/default/security/api/_upgrade_check.yaml new file mode 100644 index 000000000..60d553753 --- /dev/null +++ b/tests/default/security/api/_upgrade_check.yaml @@ -0,0 +1,11 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test _upgrade_check endpoint. +version: '> 2.13' + +chapters: + - synopsis: Check whether an upgrade can be performed. + path: /_plugins/_security/api/_upgrade_check + method: GET + response: + status: 200 diff --git a/tests/default/security/api/_upgrade_perform.yaml b/tests/default/security/api/_upgrade_perform.yaml new file mode 100644 index 000000000..643f7896f --- /dev/null +++ b/tests/default/security/api/_upgrade_perform.yaml @@ -0,0 +1,14 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test _upgrade_perform endpoint. +version: '> 2.13' + +chapters: + - synopsis: Perform the upgrade. + path: /_plugins/_security/api/_upgrade_perform + method: POST + request: + payload: + config: [roles] + response: + status: 400 # Unable to upgrade, no differences found in 'roles' config. diff --git a/tests/default/security/api/account.yaml b/tests/default/security/api/account.yaml new file mode 100644 index 000000000..2fc10499f --- /dev/null +++ b/tests/default/security/api/account.yaml @@ -0,0 +1,21 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test account endpoint. + +chapters: + - synopsis: Get account details. + path: /_plugins/_security/api/account + method: GET + response: + status: 200 + # TODO: following test can be changed to interact with test user, to be created in prologues, once https://github.com/opensearch-project/opensearch-api-specification/issues/438 is implemented + # NOTE: this test should be updated to change test user's password created in epilogue. Until then this will be 403 since admin is a reserved user. + - synopsis: Change password. + path: /_plugins/_security/api/account + method: PUT + request: + payload: + current_password: myStrongPassword123! + password: myWeakPassword123! + response: + status: 403 diff --git a/tests/default/security/api/actiongroups.yaml b/tests/default/security/api/actiongroups.yaml new file mode 100644 index 000000000..0321d9022 --- /dev/null +++ b/tests/default/security/api/actiongroups.yaml @@ -0,0 +1,59 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test actiongroups endpoints. + +chapters: + - synopsis: Create action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: PUT + parameters: + action_group: test + request: + payload: + allowed_actions: [] + type: index + description: Test action group + response: + status: 201 + - synopsis: Get action groups bulk. + path: /_plugins/_security/api/actiongroups + method: GET + response: + status: 200 + - synopsis: Patch action groups bulk. + path: /_plugins/_security/api/actiongroups + method: PATCH + request: + payload: + - op: add + path: /test + value: + allowed_actions: ['indices:admin/create', 'indices:admin/mapping/put'] + response: + status: 200 + - synopsis: Get action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: GET + parameters: + action_group: test + response: + status: 200 + - synopsis: Patch action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: PATCH + parameters: + action_group: test + request: + payload: + - op: replace + path: /allowed_actions + value: ['indices:admin/create', 'indices:admin/mapping/get'] + response: + status: 200 + - synopsis: Delete action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: DELETE + parameters: + action_group: test + response: + status: 200 diff --git a/tests/default/security/api/allowlist.yaml b/tests/default/security/api/allowlist.yaml new file mode 100644 index 000000000..6808131d8 --- /dev/null +++ b/tests/default/security/api/allowlist.yaml @@ -0,0 +1,31 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test allowlist endpoints. +version: '> 2.0' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Create an allowlist. + path: /_plugins/_security/api/allowlist + method: PUT + request: + payload: + enabled: true + requests: + /_cat/nodes: [GET] + response: + status: 403 + - synopsis: Get an allowlist. + path: /_plugins/_security/api/allowlist + method: GET + response: + status: 403 + - synopsis: Patch an allowlist. + path: /_plugins/_security/api/allowlist + method: PATCH + request: + payload: + - op: remove + path: /config/requests + response: + status: 403 diff --git a/tests/default/security/api/audit.yaml b/tests/default/security/api/audit.yaml new file mode 100644 index 000000000..7c004b24b --- /dev/null +++ b/tests/default/security/api/audit.yaml @@ -0,0 +1,20 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test audit endpoints. + +chapters: + - synopsis: Get an audit config. + path: /_plugins/_security/api/audit + method: GET + response: + status: 200 + - synopsis: Patch an audit config. + path: /_plugins/_security/api/audit + method: PATCH + request: + payload: + - op: add + path: /config/enabled + value: true + response: + status: 200 diff --git a/tests/default/security/api/audit/config.yaml b/tests/default/security/api/audit/config.yaml new file mode 100644 index 000000000..97335168c --- /dev/null +++ b/tests/default/security/api/audit/config.yaml @@ -0,0 +1,39 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test audit/config endpoint. + +chapters: + - synopsis: Create an audit config. + path: /_plugins/_security/api/audit/config + method: PUT + request: + payload: + enabled: true + audit: + ignore_users: [] + ignore_requests: [] + disabled_rest_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + disabled_transport_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + log_request_body: false + resolve_indices: false + resolve_bulk_requests: false + exclude_sensitive_headers: true + enable_transport: false + enable_rest: true + compliance: + enabled: true + write_log_diffs: false + read_watched_fields: {} + read_ignore_users: [] + write_watched_indices: [] + write_ignore_users: [] + read_metadata_only: true + write_metadata_only: true + external_config: false + internal_config: true + response: + status: 200 diff --git a/tests/default/security/api/authtoken.yaml b/tests/default/security/api/authtoken.yaml new file mode 100644 index 000000000..4da5718b2 --- /dev/null +++ b/tests/default/security/api/authtoken.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test authtoken endpoint. + +chapters: + - synopsis: Create an auth token. + path: /_plugins/_security/api/authtoken + method: POST + response: + status: 200 diff --git a/tests/default/security/api/cache.yaml b/tests/default/security/api/cache.yaml new file mode 100644 index 000000000..d1f1d6d26 --- /dev/null +++ b/tests/default/security/api/cache.yaml @@ -0,0 +1,25 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test cache endpoint. + +chapters: + - synopsis: Get cache. + path: /_plugins/_security/api/cache + method: GET + response: + status: 501 + - synopsis: Create cache. + path: /_plugins/_security/api/cache + method: POST + response: + status: 501 + - synopsis: Update cache. + path: /_plugins/_security/api/cache + method: PUT + response: + status: 501 + - synopsis: Flush cache. + path: /_plugins/_security/api/cache + method: DELETE + response: + status: 200 diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml new file mode 100644 index 000000000..f4f23e817 --- /dev/null +++ b/tests/default/security/api/certificates.yaml @@ -0,0 +1,35 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test certificates endpoints. +version: '> 2.14' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get node. + id: get_node + path: /_cat/nodes + method: GET + parameters: + h: + - id + full_id: true + response: + status: 200 + content_type: text/plain + output: + node_id: payload + - synopsis: Get all certificates. + path: /_plugins/_security/api/certificates + method: GET + parameters: + cert_type: all + response: + status: 403 + - synopsis: Get node certificates. + path: /_plugins/_security/api/certificates/{node_id} + method: GET + parameters: + node_id: ${get_node.node_id} + cert_type: all + response: + status: 403 diff --git a/tests/default/security/api/generateonbehalfoftoken.yaml b/tests/default/security/api/generateonbehalfoftoken.yaml new file mode 100644 index 000000000..41b5f1453 --- /dev/null +++ b/tests/default/security/api/generateonbehalfoftoken.yaml @@ -0,0 +1,18 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test generateonbehalfoftoken endpoint. +version: '> 2.11' + +chapters: + - synopsis: Create an On-Behalf-Of token. + # Feature is disabled by default. https://opensearch.org/docs/latest/security/access-control/authentication-tokens/#configuration + path: /_plugins/_security/api/generateonbehalfoftoken + method: POST + request: + payload: + description: Auth token for admin + service: '' + duration: '60' + response: + content_type: text/plain + status: 400 diff --git a/tests/default/security/api/internalusers.yaml b/tests/default/security/api/internalusers.yaml new file mode 100644 index 000000000..784db4fcd --- /dev/null +++ b/tests/default/security/api/internalusers.yaml @@ -0,0 +1,60 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test internalusers endpoints. + +chapters: + - synopsis: Get internal users bulk. + path: /_plugins/_security/api/internalusers + method: GET + response: + status: 200 + - synopsis: Patch internal users bulk. + path: /_plugins/_security/api/internalusers + method: PATCH + request: + payload: + - op: add + path: /test + value: + backend_roles: [admin] + response: + status: 200 + - synopsis: Create internal user. + path: /_plugins/_security/api/internalusers/{username} + method: PUT + parameters: + username: test + request: + payload: + password: myWeakPassword123! + opendistro_security_roles: [] + backend_roles: [] + attributes: {} + response: + status: 200 + - synopsis: Get internal user. + path: /_plugins/_security/api/internalusers/{username} + method: GET + parameters: + username: test + response: + status: 200 + - synopsis: Patch internal user. + path: /_plugins/_security/api/internalusers/{username} + method: PATCH + parameters: + username: test + request: + payload: + - op: add + path: /opendistro_security_roles + value: [all_access] + response: + status: 200 + - synopsis: Delete internal user. + path: /_plugins/_security/api/internalusers/{username} + method: DELETE + parameters: + username: test + response: + status: 200 diff --git a/tests/default/security/api/internalusers.yml/authtoken.yml b/tests/default/security/api/internalusers.yml/authtoken.yml new file mode 100644 index 000000000..2cfb3c6a1 --- /dev/null +++ b/tests/default/security/api/internalusers.yml/authtoken.yml @@ -0,0 +1,35 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test internalusers/authtoken endpoint. +version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 + +prologues: + - path: /_plugins/_security/api/internalusers/{username} + method: PUT + parameters: + username: test + request: + payload: + opendistro_security_roles: [] + backend_roles: [] + attributes: + service: true + enabled: true + status: [201] + +chapters: + # Auth-tokens can only be vended for service accounts. + - synopsis: Create internal user token. + path: /_plugins/_security/api/internalusers/{username}/authtoken + method: POST + parameters: + username: test + response: + status: 200 + +epilogues: + - path: /_plugins/_security/api/internalusers/{username} + method: DELETE + parameters: + username: test + status: [200] diff --git a/tests/default/security/api/migrate.yaml b/tests/default/security/api/migrate.yaml new file mode 100644 index 000000000..3cf053554 --- /dev/null +++ b/tests/default/security/api/migrate.yaml @@ -0,0 +1,11 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test migrate endpoint. + +# BAD_REQUEST. Can not migrate configuration because it was already migrated. +chapters: + - synopsis: Migrate v6 to v7 config. + path: /_plugins/_security/api/migrate + method: POST + response: + status: 400 diff --git a/tests/default/security/api/nodesdn.yaml b/tests/default/security/api/nodesdn.yaml new file mode 100644 index 000000000..0cfcd1b43 --- /dev/null +++ b/tests/default/security/api/nodesdn.yaml @@ -0,0 +1,61 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test nodesdn endpoints. + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +# The setting `plugins. security. nodes_dn_dynamic_config_enabled` must be enabled. +chapters: + - synopsis: Get distinguished names. + path: /_plugins/_security/api/nodesdn + method: GET + parameters: + show_all: true + response: + status: 400 + - synopsis: Patch distinguished names. + path: /_plugins/_security/api/nodesdn + method: PATCH + request: + payload: + - op: replace + path: /cluster1/nodes_dn/0 + value: [''] + response: + status: 400 + - synopsis: Create distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PUT + parameters: + cluster_name: test + request: + payload: + nodes_dn: + - CN=cluster3.example.com + response: + status: 400 + - synopsis: Get distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: GET + parameters: + cluster_name: test + response: + status: 400 + - synopsis: Patch distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PATCH + parameters: + cluster_name: test + request: + payload: + op: replace + path: /test/nodes_dn/0 + value: [CN=cluster2.example.com] + response: + status: 400 + - synopsis: Delete distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: DELETE + parameters: + cluster_name: test + response: + status: 400 diff --git a/tests/default/security/api/permissionsinfo.yaml b/tests/default/security/api/permissionsinfo.yaml new file mode 100644 index 000000000..59e1f9062 --- /dev/null +++ b/tests/default/security/api/permissionsinfo.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test permissionsinfo endpoint. + +chapters: + - synopsis: Get evaluated permissions for currently logged in user. + path: /_plugins/_security/api/permissionsinfo + method: GET + response: + status: 200 diff --git a/tests/default/security/api/roles.yaml b/tests/default/security/api/roles.yaml new file mode 100644 index 000000000..a79ef6312 --- /dev/null +++ b/tests/default/security/api/roles.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test roles endpoints. + +chapters: + - synopsis: Create role. + path: /_plugins/_security/api/roles/{role} + method: PUT + parameters: + role: test + request: + payload: + cluster_permissions: + - cluster_composite_ops + - indices_monitor + index_permissions: + - index_patterns: + - 'movies*' + dls: '' + fls: [] + masked_fields: [] + allowed_actions: + - read + tenant_permissions: + - tenant_patterns: + - human_resources + allowed_actions: + - kibana_all_read + response: + status: 201 + - synopsis: Get roles bulk. + path: /_plugins/_security/api/roles + method: GET + response: + status: 200 + - synopsis: Patch roles bulk. + path: /_plugins/_security/api/roles + method: PATCH + request: + payload: + - op: add + path: /test/index_permissions/0/fls + value: + - 'random*' + - ~random1 + response: + status: 200 + - synopsis: Get role. + path: /_plugins/_security/api/roles/{role} + method: GET + parameters: + role: test + response: + status: 200 + - synopsis: Patch role. + path: /_plugins/_security/api/roles/{role} + method: PATCH + parameters: + role: test + request: + payload: + - op: add + path: /index_permissions/0/fls + value: + - 'random*' + - ~random1 + response: + status: 200 + - synopsis: Delete role. + path: /_plugins/_security/api/roles/{role} + method: DELETE + parameters: + role: test + response: + status: 200 diff --git a/tests/default/security/api/rolesmapping.yaml b/tests/default/security/api/rolesmapping.yaml new file mode 100644 index 000000000..8fed57e18 --- /dev/null +++ b/tests/default/security/api/rolesmapping.yaml @@ -0,0 +1,77 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test rolesmapping endpoint. + +prologues: + - path: /_plugins/_security/api/roles/{role} + method: PUT + parameters: + role: test + request: + payload: + cluster_permissions: + - cluster_composite_ops + status: [201] +chapters: + - synopsis: Get rolesmapping bulk. + path: /_plugins/_security/api/rolesmapping + method: GET + response: + status: 200 + - synopsis: Patch rolesmapping bulk. + path: /_plugins/_security/api/rolesmapping + method: PATCH + request: + payload: + - op: add + path: /all_access + value: + users: [test] + backend_roles: [admin] + response: + status: 200 + - synopsis: Create rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: PUT + parameters: + role: test + request: + payload: + backend_roles: [captains] + hosts: + - '*.example.com' + users: [test] + response: + status: 201 + - synopsis: Get rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: GET + parameters: + role: test + response: + status: 200 + - synopsis: Patch rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: PATCH + parameters: + role: test + request: + payload: + - op: replace + path: /backend_roles + value: [admin] + response: + status: 200 + - synopsis: Delete rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: DELETE + parameters: + role: test + response: + status: 200 +epilogues: + - path: /_plugins/_security/api/roles/{role} + method: DELETE + parameters: + role: test + status: [200] diff --git a/tests/default/security/api/securityconfig.yaml b/tests/default/security/api/securityconfig.yaml new file mode 100644 index 000000000..28c6573bd --- /dev/null +++ b/tests/default/security/api/securityconfig.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test securityconfig endpoints. +version: '> 2.9' + +# ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get a security config. + path: /_plugins/_security/api/securityconfig + method: GET + response: + status: 200 + - synopsis: Patch a security config. + path: /_plugins/_security/api/securityconfig + method: PATCH + request: + payload: + - op: replace + path: /config/dynamic/authc/basic_internal_auth_domain/transport_enabled + value: true + response: + status: 403 diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml new file mode 100644 index 000000000..5bae694be --- /dev/null +++ b/tests/default/security/api/securityconfig/config.yaml @@ -0,0 +1,43 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test securityconfig/config endpoint. +version: '>2.9' + +# ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. +chapters: + - synopsis: Update a security config. + path: /_plugins/_security/api/securityconfig/config + method: PUT + request: + payload: + dynamic: + filtered_alias_mode: warn + disable_rest_auth: false + disable_intertransport_auth: false + respect_request_indices_options: false + opensearch-dashboards: + multitenancy_enabled: true + server_username: kibanaserver + index: .opensearch-dashboards + http: + anonymous_auth_enabled: false + authc: + basic_internal_auth_domain: + http_enabled: true + transport_enabled: true + order: 0 + http_authenticator: + challenge: true + type: basic + config: {} + authentication_backend: + type: intern + config: {} + description: Authenticate via HTTP Basic against internal users database + auth_failure_listeners: {} + do_not_fail_on_forbidden: false + multi_rolespan_enabled: true + hosts_resolver_mode: ip-only + do_not_fail_on_forbidden_empty: false + response: + status: 403 diff --git a/tests/default/security/api/ssl/certs.yml b/tests/default/security/api/ssl/certs.yml new file mode 100644 index 000000000..168ff0964 --- /dev/null +++ b/tests/default/security/api/ssl/certs.yml @@ -0,0 +1,12 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/certs endpoint. +version: '>= 2.0' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get ssl certificates. + path: /_plugins/_security/api/ssl/certs + method: GET + response: + status: 403 diff --git a/tests/default/security/api/ssl/http/reloadcerts.yaml b/tests/default/security/api/ssl/http/reloadcerts.yaml new file mode 100644 index 000000000..30f1f043b --- /dev/null +++ b/tests/default/security/api/ssl/http/reloadcerts.yaml @@ -0,0 +1,12 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/http/reloadcerts endpoint. +version: '> 2.7' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Reload http certs. + path: /_plugins/_security/api/ssl/http/reloadcerts + method: PUT + response: + status: 403 diff --git a/tests/default/security/api/ssl/transport/reloadcerts.yaml b/tests/default/security/api/ssl/transport/reloadcerts.yaml new file mode 100644 index 000000000..9585b1a17 --- /dev/null +++ b/tests/default/security/api/ssl/transport/reloadcerts.yaml @@ -0,0 +1,12 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/transport/reloadcerts endpoint. +version: '> 2.7' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Reload transport certs. + path: /_plugins/_security/api/ssl/transport/reloadcerts + method: PUT + response: + status: 403 diff --git a/tests/default/security/api/tenancy/config.yaml b/tests/default/security/api/tenancy/config.yaml new file mode 100644 index 000000000..1e3d95262 --- /dev/null +++ b/tests/default/security/api/tenancy/config.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test tenancy/config endpoints. +version: '> 2.6' + +chapters: + - synopsis: Get tenancy config. + path: /_plugins/_security/api/tenancy/config + method: GET + response: + status: 200 + - synopsis: Create or Update tenancy config. + path: /_plugins/_security/api/tenancy/config + method: PUT + request: + payload: + default_tenant: admin_tenant + private_tenant_enabled: false + multitenancy_enabled: true + sign_in_options: [] + response: + status: 200 diff --git a/tests/default/security/api/tenants.yaml b/tests/default/security/api/tenants.yaml new file mode 100644 index 000000000..3e41617ee --- /dev/null +++ b/tests/default/security/api/tenants.yaml @@ -0,0 +1,56 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test tenants endpoints. + +chapters: + - synopsis: Create tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: PUT + parameters: + tenant: test + request: + payload: + description: A test tenant. + response: + status: 201 + - synopsis: Get tenants bulk. + path: /_plugins/_security/api/tenants + method: GET + response: + status: 200 + - synopsis: Patch tenants bulk. + path: /_plugins/_security/api/tenants + method: PATCH + request: + payload: + - op: replace + path: /test/description + value: A very good description + response: + status: 200 + - synopsis: Get tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: GET + parameters: + tenant: test + response: + status: 200 + - synopsis: Patch tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: PATCH + parameters: + tenant: test + request: + payload: + - op: replace + path: /description + value: An updated description + response: + status: 200 + - synopsis: Delete tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: DELETE + parameters: + tenant: test + response: + status: 200 diff --git a/tests/default/security/api/user.yaml b/tests/default/security/api/user.yaml new file mode 100644 index 000000000..4c676f39a --- /dev/null +++ b/tests/default/security/api/user.yaml @@ -0,0 +1,37 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test user endpoints. + +chapters: + - synopsis: Get user bulk. + path: /_plugins/_security/api/user + method: GET + response: + status: 200 + - synopsis: Create user. + path: /_plugins/_security/api/user/{username} + method: PUT + parameters: + username: test + request: + payload: + password: myWeakPassword123! + opendistro_security_roles: [] + backend_roles: [] + attributes: {} + response: + status: 201 + - synopsis: Get user. + path: /_plugins/_security/api/user/{username} + method: GET + parameters: + username: test + response: + status: 200 + - synopsis: Delete user. + path: /_plugins/_security/api/user/{username} + method: DELETE + parameters: + username: test + response: + status: 200 diff --git a/tests/default/security/api/user/authtoken.yaml b/tests/default/security/api/user/authtoken.yaml new file mode 100644 index 000000000..edf450fb6 --- /dev/null +++ b/tests/default/security/api/user/authtoken.yaml @@ -0,0 +1,35 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test authtoken endpoints for user. +version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 + +prologues: + - path: /_plugins/_security/api/user/{username} + method: PUT + parameters: + username: test + request: + payload: + opendistro_security_roles: [] + backend_roles: [] + attributes: + service: true + enabled: true + status: [201] + +chapters: + # Auth-tokens can only be vended for service accounts. + - synopsis: Create user token. + path: /_plugins/_security/api/user/{username}/authtoken + method: POST + parameters: + username: test + response: + status: 501 + +epilogues: + - path: /_plugins/_security/api/user/{username} + method: DELETE + parameters: + username: test + status: [200] diff --git a/tests/default/security/api/validate.yaml b/tests/default/security/api/validate.yaml new file mode 100644 index 000000000..5c4cbdfc4 --- /dev/null +++ b/tests/default/security/api/validate.yaml @@ -0,0 +1,13 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test validate endpoint. + +# BAD_REQUEST. Can not migrate configuration because it was already migrated. +chapters: + - synopsis: Check whether v6 configuration is valid. + path: /_plugins/_security/api/validate + method: GET + parameters: + accept_invalid: false + response: + status: 400 diff --git a/tests/default/security/authinfo.yaml b/tests/default/security/authinfo.yaml new file mode 100644 index 000000000..1d65c31f0 --- /dev/null +++ b/tests/default/security/authinfo.yaml @@ -0,0 +1,34 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test authinfo endpoint. + +chapters: + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: < 2.13 + response: + status: 200 + - synopsis: Get auth info via POST. + path: /_plugins/_security/authinfo + method: POST + version: < 2.13 + response: + status: 200 + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: = 2.13 + parameters: + verbose: false + response: + status: 200 + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: '> 2.13' + parameters: + verbose: false + auth_type: basic + response: + status: 200 diff --git a/tests/default/security/dashboardsinfo.yaml b/tests/default/security/dashboardsinfo.yaml new file mode 100644 index 000000000..fb28a9914 --- /dev/null +++ b/tests/default/security/dashboardsinfo.yaml @@ -0,0 +1,15 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test dashboardsinfo endpoint. + +chapters: + - synopsis: Get dashboards info. + path: /_plugins/_security/dashboardsinfo + method: GET + response: + status: 200 + - synopsis: Get dashboards info via POST. + path: /_plugins/_security/dashboardsinfo + method: POST + response: + status: 200 diff --git a/tests/default/security/health.yaml b/tests/default/security/health.yaml new file mode 100644 index 000000000..b1c42b037 --- /dev/null +++ b/tests/default/security/health.yaml @@ -0,0 +1,27 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test health endpoint. + +chapters: + - synopsis: Get security health info. + path: /_plugins/_security/health + method: GET + parameters: + mode: strict + response: + status: 200 + payload: + message: null + mode: strict + status: UP + - synopsis: Get security health info via POST. + path: /_plugins/_security/health + method: POST + parameters: + mode: strict + response: + status: 200 + payload: + message: null + mode: strict + status: UP diff --git a/tests/default/security/sslinfo.yaml b/tests/default/security/sslinfo.yaml new file mode 100644 index 000000000..b73d18ee4 --- /dev/null +++ b/tests/default/security/sslinfo.yaml @@ -0,0 +1,25 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test sslinfo endpoint. + +chapters: + - synopsis: Get ssl info. + path: /_opendistro/_security/sslinfo + method: GET + parameters: + show_dn: false + response: + status: 200 + payload: + principal: null + peer_certificates: '0' + ssl_protocol: TLSv1.3 + ssl_openssl_available: false + ssl_openssl_version: -1 + ssl_openssl_version_string: null + ssl_openssl_non_available_cause: 'java.lang.ClassNotFoundException: io.netty.internal.tcnative.SSLContext' + ssl_openssl_supports_key_manager_factory: false + ssl_openssl_supports_hostname_validation: false + ssl_provider_http: JDK + ssl_provider_transport_server: JDK + ssl_provider_transport_client: JDK diff --git a/tests/default/security/tenantinfo.yaml b/tests/default/security/tenantinfo.yaml new file mode 100644 index 000000000..8b0aeee54 --- /dev/null +++ b/tests/default/security/tenantinfo.yaml @@ -0,0 +1,17 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test tenantinfo endpoint. + +chapters: + - synopsis: Get tenant info. + path: /_plugins/_security/tenantinfo + method: GET + response: + status: 403 # only allowed for super-admin or dashboards-server role mapping + content_type: text/plain + - synopsis: Get tenant info via POST. + path: /_plugins/_security/tenantinfo + method: POST + response: + status: 403 # only allowed for super-admin or dashboards-server role mapping + content_type: text/plain diff --git a/tests/default/security/whoami.yaml b/tests/default/security/whoami.yaml new file mode 100644 index 000000000..3e2c5016b --- /dev/null +++ b/tests/default/security/whoami.yaml @@ -0,0 +1,24 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test whoami endpoints. +version: '>= 2.0' + +chapters: + - synopsis: Get current user info. + path: /_plugins/_security/whoami + method: GET + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false + - synopsis: Get current user info via POST. + path: /_plugins/_security/whoami + method: POST + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false diff --git a/tests/default/security/whoamiprotected.yaml b/tests/default/security/whoamiprotected.yaml new file mode 100644 index 000000000..a3771fb7c --- /dev/null +++ b/tests/default/security/whoamiprotected.yaml @@ -0,0 +1,15 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test whoamiprotected endpoint. +version: '> 2.10' + +chapters: + - synopsis: Get current user info from protected endpoint. + path: /_plugins/_security/whoamiprotected + method: GET + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false From 08bbc0e2205c59b70f77620ce47a49333cc45f70 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 17:10:36 -0400 Subject: [PATCH 24/31] Fix: correctly named folders/files. (#497) * Fix: correctly named folders/files. Signed-off-by: dblock * Refresh index to avoid flaky test. Signed-off-by: dblock --------- Signed-off-by: dblock --- tests/default/indices/{mapping.yml => mapping.yaml} | 0 tests/default/observability/observability.yaml | 5 ++++- .../authtoken.yml => internalusers/authtoken.yaml} | 0 tests/default/security/api/ssl/{certs.yml => certs.yaml} | 0 .../security/api/{_upgrade_check.yaml => upgrade_check.yaml} | 0 .../api/{_upgrade_perform.yaml => upgrade_perform.yaml} | 0 6 files changed, 4 insertions(+), 1 deletion(-) rename tests/default/indices/{mapping.yml => mapping.yaml} (100%) rename tests/default/security/api/{internalusers.yml/authtoken.yml => internalusers/authtoken.yaml} (100%) rename tests/default/security/api/ssl/{certs.yml => certs.yaml} (100%) rename tests/default/security/api/{_upgrade_check.yaml => upgrade_check.yaml} (100%) rename tests/default/security/api/{_upgrade_perform.yaml => upgrade_perform.yaml} (100%) diff --git a/tests/default/indices/mapping.yml b/tests/default/indices/mapping.yaml similarity index 100% rename from tests/default/indices/mapping.yml rename to tests/default/indices/mapping.yaml diff --git a/tests/default/observability/observability.yaml b/tests/default/observability/observability.yaml index f9e37461c..e27b2be5e 100644 --- a/tests/default/observability/observability.yaml +++ b/tests/default/observability/observability.yaml @@ -57,6 +57,9 @@ prologues: - name: field1 type: text status: [200] + - path: /_refresh + method: POST + status: [200] chapters: - synopsis: Retrieve specific Observability object after creation. path: /_plugins/_observability/object/{object_id} @@ -144,7 +147,7 @@ chapters: status: 200 payload: startIndex: 0 - totalHits: 0 + totalHits: 1 totalHitRelation: eq observabilityObjectList: [] epilogues: diff --git a/tests/default/security/api/internalusers.yml/authtoken.yml b/tests/default/security/api/internalusers/authtoken.yaml similarity index 100% rename from tests/default/security/api/internalusers.yml/authtoken.yml rename to tests/default/security/api/internalusers/authtoken.yaml diff --git a/tests/default/security/api/ssl/certs.yml b/tests/default/security/api/ssl/certs.yaml similarity index 100% rename from tests/default/security/api/ssl/certs.yml rename to tests/default/security/api/ssl/certs.yaml diff --git a/tests/default/security/api/_upgrade_check.yaml b/tests/default/security/api/upgrade_check.yaml similarity index 100% rename from tests/default/security/api/_upgrade_check.yaml rename to tests/default/security/api/upgrade_check.yaml diff --git a/tests/default/security/api/_upgrade_perform.yaml b/tests/default/security/api/upgrade_perform.yaml similarity index 100% rename from tests/default/security/api/_upgrade_perform.yaml rename to tests/default/security/api/upgrade_perform.yaml From 2ee31047f57d3daa50cf1be72c75533683cb498f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 17:31:46 -0400 Subject: [PATCH 25/31] Skip only known or hidden files. (#498) Signed-off-by: dblock --- tools/src/tester/TestRunner.ts | 7 +++---- tools/tests/tester/fixtures/stories/.ignore-dot-file | 1 + tools/tests/tester/fixtures/stories/docker-compose.yml | 1 + tools/tests/tester/fixtures/stories/ignore.wrong.extension | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 tools/tests/tester/fixtures/stories/.ignore-dot-file create mode 100644 tools/tests/tester/fixtures/stories/docker-compose.yml delete mode 100644 tools/tests/tester/fixtures/stories/ignore.wrong.extension diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0c0f3c4ff..0b197d648 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -64,10 +64,9 @@ export default class TestRunner { #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (fs.statSync(path).isFile()) { - if (!path.endsWith('.yaml')) { - return [] - } + if (file.startsWith('.') || file == 'docker-compose.yml') { + return [] + } else if (fs.statSync(path).isFile()) { const story: Story = read_yaml(path) return [{ display_path: next_prefix === '' ? basename(path) : next_prefix, diff --git a/tools/tests/tester/fixtures/stories/.ignore-dot-file b/tools/tests/tester/fixtures/stories/.ignore-dot-file new file mode 100644 index 000000000..11c2f4cfd --- /dev/null +++ b/tools/tests/tester/fixtures/stories/.ignore-dot-file @@ -0,0 +1 @@ +Hidden file, ignored. \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/docker-compose.yml b/tools/tests/tester/fixtures/stories/docker-compose.yml new file mode 100644 index 000000000..a47276c99 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/docker-compose.yml @@ -0,0 +1 @@ +# A docker-compose.yml file, ignored. diff --git a/tools/tests/tester/fixtures/stories/ignore.wrong.extension b/tools/tests/tester/fixtures/stories/ignore.wrong.extension deleted file mode 100644 index a649676ae..000000000 --- a/tools/tests/tester/fixtures/stories/ignore.wrong.extension +++ /dev/null @@ -1 +0,0 @@ -Not a .yaml file, should be ignored From 42fd4d7365320b29a14ea374da284066b07d5de1 Mon Sep 17 00:00:00 2001 From: Theo Nam Truong Date: Mon, 12 Aug 2024 16:08:26 -0600 Subject: [PATCH 26/31] Fixed schema refs for ml.yaml (#489) * Fixed schema refs for ml.yaml Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong --------- Signed-off-by: Theo Truong --- spec/namespaces/ml.yaml | 32 ++++++++++++++++++++++++-------- spec/schemas/ml._common.yaml | 5 +++-- tests/default/ml/models.yaml | 18 +++++++++++++----- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 54b5ea532..2822ceb89 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -130,36 +130,52 @@ components: ml.search_models: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsQuery' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsQuery' responses: ml.register_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroupRegistration' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroupRegistration' ml.get_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' ml.delete_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' ml.register_model@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/Task' + schema: + type: object + properties: + status: + type: string + task_id: + type: string + required: + - status + - task_id ml.delete_model@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' ml.get_task@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/Task' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/Task' ml.search_models@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsResponse' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsResponse' parameters: ml.get_model_group::path.model_group_id: name: model_group_id diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 3e06e2612..8d1b40d3e 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -108,10 +108,10 @@ components: properties: model_id: type: string - description: The model ID. + task_id: + type: string state: type: string - description: The state. enum: - CANCELLED - COMPLETED @@ -124,6 +124,7 @@ components: description: Task type. enum: - DEPLOY_MODEL + - REGISTER_MODEL function_name: type: string worker_node: diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index ac386c0a4..4bdac64e1 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -12,8 +12,8 @@ prologues: ml_commons: only_run_on_ml_node: false chapters: - - synopsis: Create model. - id: create_model + - synopsis: Register model. + id: register_model path: /_plugins/_ml/models/_register method: POST request: @@ -25,17 +25,25 @@ chapters: status: 200 output: task_id: payload.task_id - - synopsis: Wait for task. + - synopsis: Wait to get completed task. + id: get_completed_task path: /_plugins/_ml/tasks/{task_id} method: GET warnings: multiple-paths-detected: false parameters: - task_id: ${create_model.task_id} + task_id: ${register_model.task_id} response: status: 200 payload: state: COMPLETED + output: + model_id: payload.model_id retry: - count: 3 + count: 5 wait: 30000 + - synopsis: Delete model. + path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_task.model_id} + method: DELETE From 09824d0d0913a27ed9deeedf71fa4fffb1bed8ff Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 10:58:38 -0400 Subject: [PATCH 27/31] Add support for ML neural search. (#504) Signed-off-by: dblock --- .cspell | 6 +- CHANGELOG.md | 3 +- spec/namespaces/ml.yaml | 52 +++++ spec/schemas/_common.mapping.yaml | 36 ++++ spec/schemas/_common.query_dsl.yaml | 28 +++ spec/schemas/ml._common.yaml | 14 ++ .../ingest/pipeline/neural_search.yaml | 194 ++++++++++++++++++ .../text_embedding.yaml} | 2 +- 8 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 tests/default/ingest/pipeline/neural_search.yaml rename tests/default/ingest/{pipeline.yaml => pipeline/text_embedding.yaml} (94%) diff --git a/.cspell b/.cspell index 1fcd3199c..65a801cbf 100644 --- a/.cspell +++ b/.cspell @@ -59,6 +59,7 @@ gsub Gsub haasephonetik heteroscedastic +hnsw homoscedastic hotthreads huggingface @@ -74,6 +75,7 @@ kstem kuromoji Kuromoji languageset +localstats Lovins lucene Lucene @@ -176,6 +178,7 @@ tokenfilters translog Translog tubone +Undeploys unigrams Unmanaged unmatch @@ -186,5 +189,4 @@ urldecode vectory whoamiprotected wordnet -Yrtsd -localstats \ No newline at end of file +Yrtsd \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a085e8a..e1f7db7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) - Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) -- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) ### Changed diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 2822ceb89..00054b4cb 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -53,6 +53,26 @@ paths: responses: '200': $ref: '#/components/responses/ml.delete_model@200' + /_plugins/_ml/models/{model_id}/_deploy: + post: + operationId: ml.deploy_model.0 + x-operation-group: ml.deploy_model + description: Deploys a model. + parameters: + - $ref: '#/components/parameters/ml.deploy_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.deploy_model@200' + /_plugins/_ml/models/{model_id}/_undeploy: + post: + operationId: ml.undeploy_model.0 + x-operation-group: ml.undeploy_model + description: Undeploys a model. + parameters: + - $ref: '#/components/parameters/ml.undeploy_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.undeploy_model@200' /_plugins/_ml/tasks/{task_id}: get: operationId: ml.get_task.0 @@ -161,6 +181,26 @@ components: required: - status - task_id + ml.deploy_model@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + task_id: + type: string + task_type: + type: string + required: + - status + - task_id + ml.undeploy_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/UndeployModelResponse' ml.delete_model@200: content: application/json: @@ -195,6 +235,18 @@ components: required: true schema: type: string + ml.deploy_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string + ml.undeploy_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.get_task::path.task_id: name: task_id in: path diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index cc8e8de0a..ef7a31a31 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -208,6 +208,7 @@ components: - $ref: '#/components/schemas/IntegerRangeProperty' - $ref: '#/components/schemas/IpRangeProperty' - $ref: '#/components/schemas/LongRangeProperty' + - $ref: '#/components/schemas/KnnVectorProperty' BinaryProperty: allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' @@ -1129,6 +1130,30 @@ components: type: boolean index: type: boolean + KnnVectorPropertyBase: + type: object + properties: + dimension: + type: number + method: + $ref: '#/components/schemas/KnnVectorMethod' + required: + - dimension + KnnVectorMethod: + type: object + properties: + name: + type: string + space_type: + type: string + engine: + type: string + parameters: + type: object + additionalProperties: + type: object + required: + - name DoubleRangeProperty: allOf: - $ref: '#/components/schemas/RangePropertyBase' @@ -1184,6 +1209,17 @@ components: - long_range required: - type + KnnVectorProperty: + allOf: + - $ref: '#/components/schemas/KnnVectorPropertyBase' + - type: object + properties: + type: + type: string + enum: + - knn_vector + required: + - type MatchType: type: string enum: diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index a0d547cd5..a4bb57b06 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -112,6 +112,8 @@ components: $ref: '#/components/schemas/MultiMatchQuery' nested: $ref: '#/components/schemas/NestedQuery' + neural: + $ref: '#/components/schemas/NeuralQuery' parent_id: $ref: '#/components/schemas/ParentIdQuery' percolate: @@ -1222,6 +1224,32 @@ components: required: - path - query + NeuralQuery: + allOf: + - $ref: '#/components/schemas/QueryBase' + - type: object + additionalProperties: + $ref: '#/components/schemas/NeuralQueryVectorField' + NeuralQueryVectorField: + type: object + properties: + query_text: + type: string + query_image: + type: string + format: binary + model_id: + type: string + k: + type: integer + min_score: + type: number + max_distance: + type: number + filter: + $ref: '#/components/schemas/QueryContainer' + required: + - model_id ParentIdQuery: allOf: - $ref: '#/components/schemas/QueryBase' diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 8d1b40d3e..400a92fd3 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -139,5 +139,19 @@ components: format: int64 is_async: type: boolean + error: + type: string required: - state + UndeployModelResponse: + type: object + additionalProperties: + $ref: '#/components/schemas/UndeployModelResponseModels' + UndeployModelResponseModels: + type: object + properties: + stats: + $ref: '#/components/schemas/UndeployModelResponseStats' + UndeployModelResponseStats: + type: object + additionalProperties: true diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml new file mode 100644 index 000000000..0d1d89a22 --- /dev/null +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -0,0 +1,194 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation a neural search ingest pipeline. +prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins: + ml_commons: + only_run_on_ml_node: false +epilogues: + - path: /_ingest/pipeline/movies_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] + - path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + status: [200, 404] + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] + - path: /_plugins/_ml/model_groups/{model_group_id} + method: DELETE + status: [200, 404] + parameters: + model_group_id: ${create_model_group.test_model_group_id} +version: '>= 2.11' +chapters: + - synopsis: Create model group. + id: create_model_group + path: /_plugins/_ml/model_groups/_register + method: POST + request: + payload: + name: NLP_Group + description: Model group for NLP models. + response: + status: 200 + output: + test_model_group_id: payload.model_group_id + - synopsis: Register model. + id: register_model + path: /_plugins/_ml/models/_register + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + response: + status: 200 + output: + task_id: payload.task_id + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Create ingest pipeline for text embedding. + path: /_ingest/pipeline/{id} + method: PUT + parameters: + id: movies_pipeline + request: + payload: + description: Extracts text from field and embeds it. + processors: + - text_embedding: + model_id: text-embedding-model + field_map: + text: passage_embedding + response: + status: 200 + payload: + acknowledged: true + - synopsis: Create an index using the pipeline. + path: /{index} + method: PUT + parameters: + index: movies + request: + payload: + settings: + index.knn: true + default_pipeline: movies_pipeline + mappings: + properties: + title: + type: text + year: + type: integer + passage_embedding: + type: knn_vector + dimension: 768 + method: + engine: lucene + space_type: l2 + name: hnsw + parameters: {} + response: + status: 200 + payload: + acknowledged: true + - synopsis: Ingest data. + path: /_bulk + method: POST + parameters: + refresh: 'true' + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {author: Nicolas Winding Refn, title: Drive, year: 1960} + response: + status: 200 + - synopsis: Search. + path: /{index}/_search + method: POST + parameters: + index: movies + request: + payload: + _source: + excludes: [passage_embedding] + query: + bool: + should: + - script_score: + query: + neural: + passage_embedding: + query_text: Money + model_id: ${get_completed_register_model_task.model_id} + k: 100 + script: + source: _score * 1.5 + - script_score: + query: + match: + title: Moneyball + script: + source: _score * 1.7 + - synopsis: Undeploy a model. + path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + response: + status: 200 diff --git a/tests/default/ingest/pipeline.yaml b/tests/default/ingest/pipeline/text_embedding.yaml similarity index 94% rename from tests/default/ingest/pipeline.yaml rename to tests/default/ingest/pipeline/text_embedding.yaml index 3145dcb62..6ba46e8e9 100644 --- a/tests/default/ingest/pipeline.yaml +++ b/tests/default/ingest/pipeline/text_embedding.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test the creation of an ingest pipeline with a text embedding processor. epilogues: From 9d3bc340ccd7d049e7d6e14a4aff2293780cb446 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 17:53:00 -0400 Subject: [PATCH 28/31] Remove support for comma-separated semver range. (#501) Signed-off-by: dblock --- TESTING_GUIDE.md | 6 +-- .../ingest/pipeline/text_embedding.yaml | 2 +- .../security/api/securityconfig/config.yaml | 2 +- tools/src/_utils/semver.ts | 20 ++++++++ tools/src/merger/OpenApiVersionExtractor.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 2 +- tools/tests/_utils/semver.test.ts | 51 +++++++++++++++++++ tools/tests/tester/fixtures/evals/passed.yaml | 19 +++++++ .../tests/tester/fixtures/stories/passed.yaml | 6 +++ 9 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 tools/src/_utils/semver.ts create mode 100644 tools/tests/_utils/semver.test.ts diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 61f6c2828..1328cc863 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -195,11 +195,11 @@ You can also reuse output in payload expectations. See [tests/plugins/index_stat ### Managing Versions -It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. +It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver or a semver range in your test stories or chapters as follows. ```yaml -- synopsis: Search with `phase_took` added in OpenSearch 2.12. - version: '>= 2.12' +- synopsis: Search with `phase_took` added in OpenSearch 2.12 and removed in version 3. + version: '>=2.12 <3' path: /{index}/_search parameters: index: movies diff --git a/tests/default/ingest/pipeline/text_embedding.yaml b/tests/default/ingest/pipeline/text_embedding.yaml index 6ba46e8e9..fe70aee21 100644 --- a/tests/default/ingest/pipeline/text_embedding.yaml +++ b/tests/default/ingest/pipeline/text_embedding.yaml @@ -5,7 +5,7 @@ epilogues: - path: /_ingest/pipeline/books_pipeline method: DELETE status: [200, 404] -version: '>= 2.11, < 3.0' # TODO: re-enable using a 3.0 build with the neural-search plugin +version: '>= 2.11' chapters: - synopsis: Create ingest pipeline for text embedding. path: /_ingest/pipeline/{id} diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml index 5bae694be..3f51b38c3 100644 --- a/tests/default/security/api/securityconfig/config.yaml +++ b/tests/default/security/api/securityconfig/config.yaml @@ -1,7 +1,7 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test securityconfig/config endpoint. -version: '>2.9' +version: '> 2.9' # ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. chapters: diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts new file mode 100644 index 000000000..52de7e899 --- /dev/null +++ b/tools/src/_utils/semver.ts @@ -0,0 +1,20 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from 'semver' + +export function coerce(version: string) : string { + return semver.coerce(version)?.toString() ?? version +} + +export function satisfies(version: string | semver.SemVer | undefined, range: string): boolean { + if (version === undefined || version === '') return true + if (semver.validRange(range) == null) throw `Invalid semver ${range}.` + return semver.satisfies(version, range) +} diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 5d4164b1c..3f2fb396b 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -11,7 +11,7 @@ import _, { extend, isEmpty } from 'lodash' import { delete_matching_keys, find_refs, write_yaml } from '../helpers' import { Logger } from '../Logger' import { type OpenAPIV3 } from 'openapi-types' -import semver from 'semver' +import * as semver from '../_utils/semver' // Extract a versioned API export default class OpenApiVersionExtractor { @@ -22,7 +22,7 @@ export default class OpenApiVersionExtractor { constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { this._source_spec = source_spec - this._target_version = semver.coerce(target_version)?.toString() ?? target_version + this._target_version = semver.coerce(target_version) this._logger = logger this._spec = undefined } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index fede2e397..9020968fa 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -14,7 +14,7 @@ import { overall_result } from './helpers' import { StoryOutputs } from './StoryOutputs' import SupplementalChapterEvaluator from './SupplementalChapterEvaluator' import { ChapterOutput } from './ChapterOutput' -import * as semver from 'semver' +import * as semver from '../_utils/semver' import _ from 'lodash' export default class StoryEvaluator { diff --git a/tools/tests/_utils/semver.test.ts b/tools/tests/_utils/semver.test.ts new file mode 100644 index 000000000..2454549b7 --- /dev/null +++ b/tools/tests/_utils/semver.test.ts @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from "../../src/_utils/semver"; + +describe('coerce', () => { + it ('null', () => { + expect(semver.coerce('')).toEqual('') + expect(semver.coerce('1.2.3')).toEqual('1.2.3') + expect(semver.coerce('1.2')).toEqual('1.2.0') + expect(semver.coerce('1')).toEqual('1.0.0') + }) +}); + +describe('satisfies', () => { + it ('defaults', () => { + expect(semver.satisfies('', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('semver', () => { + expect(semver.satisfies(semver.coerce('2.17.0'), '>= 1.3 < 99.0')).toBe(true) + }) + + it ('~', () => { + expect(semver.satisfies('2.17.0', '~> 2.x')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 1.x')).toBe(false) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.18')).toBe(false) + }) + + it ('> <', () => { + expect(semver.satisfies('2.17.0', '> 2.999.0')).toBe(false) + expect(semver.satisfies('2.17.0', '< 3.0')).toBe(true) + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('ranges', () => { + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('invalid', () => { + expect(() => { semver.satisfies('1.2.3', '>= 1, < 2') }).toThrow('Invalid semver >= 1, < 2.') + }) +}); diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index 12c1b2c32..abe502379 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -176,6 +176,25 @@ chapters: overall: result: SKIPPED message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. + - title: This GET /_cat/health should run (>= 1.3, < 99.0). + overall: + result: PASSED + path: GET /_cat/health + request: + parameters: + format: + result: PASSED + request: + result: PASSED + response: + status: + result: PASSED + payload_body: + result: PASSED + payload_schema: + result: PASSED + output_values: + result: SKIPPED epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml index b4b08b963..a882666b3 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/passed.yaml @@ -89,3 +89,9 @@ chapters: path: /_cat/health parameters: format: json + - synopsis: This GET /_cat/health should run (>= 1.3, < 99.0). + version: '>= 1.3 < 99.0' + method: GET + path: /_cat/health + parameters: + format: json From 83eab434d08d5b623c8518ac50b88331d146a783 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 20:09:24 -0400 Subject: [PATCH 29/31] Added search response processors. (#505) Signed-off-by: dblock --- .cspell | 4 +- CHANGELOG.md | 1 + spec/schemas/search_pipeline._common.yaml | 215 +++++++++++++++++- .../request_processor/filter_query.yaml | 71 ++++++ .../response_processor/rename_field.yaml | 62 +++++ .../pipeline/response_processor/sort.yaml | 75 ++++++ .../ingest/pipeline/neural_search.yaml | 2 +- 7 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 tests/default/_core/search/pipeline/request_processor/filter_query.yaml create mode 100644 tests/default/_core/search/pipeline/response_processor/rename_field.yaml create mode 100644 tests/default/_core/search/pipeline/response_processor/sort.yaml diff --git a/.cspell b/.cspell index 65a801cbf..3656f245c 100644 --- a/.cspell +++ b/.cspell @@ -91,7 +91,6 @@ mmapfs mmdb mokotoff Moneyball -Moneyball msearch msmarco mtermvectors @@ -138,6 +137,9 @@ Reindex relo reloadcerts remotestore +rerank +Rerank +Reranker rethrottle Rethrottle rolesmapping diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f7db7db..f99a07a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) +- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) ### Changed diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index 94fd37470..ebb14ee9a 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -23,7 +23,7 @@ components: response_processors: type: array items: - $ref: '#/components/schemas/RequestProcessor' + $ref: '#/components/schemas/ResponseProcessor' phase_results_processors: type: array items: @@ -187,6 +187,219 @@ components: type: string required: - sample_factor + ResponseProcessor: + oneOf: + - type: object + title: personalize_search_ranking + properties: + personalize_search_ranking: + $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor' + required: + - personalize_search_ranking + - type: object + title: retrieval_augmented_generation + properties: + retrieval_augmented_generation: + $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor' + required: + - retrieval_augmented_generation + - type: object + title: rename_field + properties: + rename_field: + $ref: '#/components/schemas/RenameFieldResponseProcessor' + required: + - rename_field + - type: object + title: rerank + properties: + rerank: + $ref: '#/components/schemas/RerankResponseProcessor' + required: + - rerank + - type: object + title: collapse + properties: + collapse: + $ref: '#/components/schemas/CollapseResponseProcessor' + required: + - collapse + - type: object + title: truncate_hits + properties: + truncate_hits: + $ref: '#/components/schemas/TruncateHitsResponseProcessor' + required: + - truncate_hits + - type: object + title: sort + properties: + sort: + $ref: '#/components/schemas/SortResponseProcessor' + required: + - sort + - type: object + title: split + properties: + split: + $ref: '#/components/schemas/SplitResponseProcessor' + required: + - split + PersonalizeSearchRankingResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + campaign_arn: + type: string + recipe: + type: string + weight: + type: number + format: float + item_id_field: + type: string + iam_role_arn: + type: string + required: + - campaign_arn + - recipe + - weight + RetrievalAugmentedGenerationResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + model_id: + type: string + context_field_list: + type: array + items: + type: string + system_prompt: + type: string + user_instructions: + type: string + required: + - context_field_list + - model_id + RenameFieldResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + target_field: + type: string + required: + - field + - target_field + RerankContext: + type: object + properties: + document_fields: + type: array + items: + type: string + required: + - document_fields + MLOpenSearchReranker: + type: object + properties: + model_id: + type: string + required: + - model_id + RerankResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + ml_opensearch: + $ref: '#/components/schemas/MLOpenSearchReranker' + context: + $ref: '#/components/schemas/RerankContext' + CollapseResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + context_prefix: + type: string + required: + - field + TruncateHitsResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + target_size: + type: integer + format: int32 + context_prefix: + type: string + SortResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + order: + type: string + target_field: + type: string + required: + - field + SplitResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + separator: + type: string + preserve_trailing: + type: boolean + target_field: + type: string + required: + - field + - separator PhaseResultsProcessor: oneOf: - type: object diff --git a/tests/default/_core/search/pipeline/request_processor/filter_query.yaml b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml new file mode 100644 index 000000000..e2d2cfb79 --- /dev/null +++ b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /_bulk + method: POST + parameters: + refresh: 'true' + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +epilogues: + - path: /_search/pipeline/filter_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: filter_pipeline + request: + payload: + request_processors: + - filter_query: + tag: tag + description: This processor restricts searches to 20th century movies. + query: + range: + year: + lte: 2000 + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: filter_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: filter_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive + director: Nicolas Winding Refn + year: 1960 diff --git a/tests/default/_core/search/pipeline/response_processor/rename_field.yaml b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml new file mode 100644 index 000000000..388d1414a --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml @@ -0,0 +1,62 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + name: Drive + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - rename_field: + field: name + target_field: title + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive diff --git a/tests/default/_core/search/pipeline/response_processor/sort.yaml b/tests/default/_core/search/pipeline/response_processor/sort.yaml new file mode 100644 index 000000000..837501f66 --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/sort.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.16' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - sort: + field: names + order: asc + target_field: sorted_names + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + sorted_names: + - '1984' + - Drive + - Moneyball \ No newline at end of file diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 0d1d89a22..2d8b41ec1 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -155,7 +155,7 @@ chapters: - {create: {_index: movies}} - {director: Bennett Miller, title: Moneyball, year: 2011} - {create: {_index: movies}} - - {author: Nicolas Winding Refn, title: Drive, year: 1960} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} response: status: 200 - synopsis: Search. From 89e383b5bbad32c2a6b002763a5d655cda4d9994 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Wed, 14 Aug 2024 06:50:50 -0500 Subject: [PATCH 30/31] Add resolve API test (#506) --- tests/default/indices/index.yaml | 3 +-- tests/default/indices/resolve.yaml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/default/indices/resolve.yaml diff --git a/tests/default/indices/index.yaml b/tests/default/indices/index.yaml index d30186c8a..5446c4735 100644 --- a/tests/default/indices/index.yaml +++ b/tests/default/indices/index.yaml @@ -84,5 +84,4 @@ chapters: version: '>= 2.0' parameters: index: books,games - cluster_manager_timeout: 10s - + cluster_manager_timeout: 10s diff --git a/tests/default/indices/resolve.yaml b/tests/default/indices/resolve.yaml new file mode 100644 index 000000000..7c0fb45ce --- /dev/null +++ b/tests/default/indices/resolve.yaml @@ -0,0 +1,20 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Tests to see if the specified index exits using the `_resolve` endpoint. +prologues: + - path: /movies + method: PUT +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: See if index `movies` exists. It should. + path: /_resolve/index/{name} + method: GET + parameters: + name: movies + expand_wildcards: none + response: + status: 200 \ No newline at end of file From 7dee041660986c5eb4bc85c2d19af006730746e9 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 14 Aug 2024 08:03:46 -0400 Subject: [PATCH 31/31] Added support for testing multiple distributions. (#483) * Added support for testing multiple distributions. Signed-off-by: dblock * Added x-distributions-included and excluded. Signed-off-by: dblock * Move node fetching to a prologue. Signed-off-by: dblock * Undo semver satisfies changes. Signed-off-by: dblock * Move OPENSEARCH_DISTRIBUTION_OPTION. Signed-off-by: dblock --------- Signed-off-by: dblock --- CHANGELOG.md | 6 ++ DEVELOPER_GUIDE.md | 4 ++ TESTING_GUIDE.md | 55 ++++++++++++++++++- json_schemas/test_story.schema.yaml | 11 ++++ spec/namespaces/_core.yaml | 2 + spec/schemas/nodes._common.yaml | 18 +++++- spec/schemas/nodes.info.yaml | 17 +----- tests/default/_core/info.yaml | 3 + .../_core/search/rest_total_hits_as_int.yaml | 1 - tests/default/cat/health.yaml | 10 ++-- tests/default/cat/indices.yaml | 4 ++ tests/default/cat/nodeattrs.yaml | 2 + tests/default/indices/cache.yaml | 2 + tests/default/indices/dangling.yaml | 2 + tests/default/indices/forcemerge.yaml | 2 + tests/default/indices/segments.yaml | 3 +- tests/default/indices/settings.yaml | 4 ++ tests/default/ml/model_groups.yaml | 2 + tests/default/ml/models.yaml | 2 + tests/default/security/api/certificates.yaml | 22 ++++---- tools/src/OpenSearchHttpClient.ts | 1 + tools/src/_utils/semver.ts | 3 +- tools/src/linter/SchemasValidator.ts | 4 +- tools/src/linter/components/OperationGroup.ts | 2 +- tools/src/merger/OpenApiVersionExtractor.ts | 38 +++++++++++-- tools/src/tester/MergedOpenApiSpec.ts | 8 ++- tools/src/tester/SchemaValidator.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 21 +++++-- tools/src/tester/TestRunner.ts | 16 ++++-- tools/src/tester/test.ts | 7 ++- tools/src/tester/types/story.types.ts | 15 ++++- tools/src/types.ts | 2 + .../merger/OpenApiVersionExtractor.test.ts | 12 ++-- .../fixtures/extractor/expected_1.3.yaml | 6 ++ .../fixtures/extractor/expected_2.0.yaml | 6 ++ tools/tests/tester/MergedOpenApiSpec.test.ts | 53 +++++++++++++++--- .../fixtures/evals/skipped/distributions.yaml | 6 ++ .../specs/complete/namespaces/index.yaml | 18 ++++++ .../stories/skipped/distributions.yaml | 9 +++ tools/tests/tester/helpers.ts | 2 +- tools/tests/tester/integ/TestRunner.test.ts | 7 ++- 41 files changed, 332 insertions(+), 80 deletions(-) create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f99a07a70..b8fd3cefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) - Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Changed @@ -116,6 +121,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Security diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 951f00cdd..43c30d7f3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -146,6 +146,10 @@ This repository includes several OpenAPI Specification Extensions to fill in any - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them. - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml). - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema. +- `x-distributions-included`: Contains a list of distributions known to include the API. +- `x-distributions-excluded`: Contains a list of distributions known to exclude the API. + +Use `opensearch.org` for the official distribution in `x-distributions-*`, `amazon-managed` for Amazon Managed OpenSearch, and `amazon-serverless` for Amazon OpenSearch Serverless. ## Writing Spec Tests diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1328cc863..d35c96aa5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -11,6 +11,7 @@ - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) - [Waiting for Tasks](#waiting-for-tasks) - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) @@ -209,7 +210,59 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne status: 200 ``` -The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. +The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. + +### Managing Distributions + +OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions-included` or `x-distributions-excluded` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not. + +```yaml +/: + get: + operationId: info.0 + x-distributions-included: + - opensearch.org + - amazon-managed + x-distributions-excluded: + - amazon-serverless + description: Returns basic information about the cluster. +``` + +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. + +```yaml +description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org +chapters: + - synopsis: Get server info. + path: / + method: GET + response: + status: 200 +``` + +To test a particular distribution pass `--opensearch-distribution` to the test tool. For example, the following runs tests against an Amazon Managed OpenSearch instance. + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export AWS_SESSION_TOKEN=... +export AWS_REGION=us-west-2 + +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec -- --opensearch-distribution=amazon-managed +``` + +The output will visible skip APIs that are not available in the `amazon-managed` distribution. + +``` +PASSED _core/bulk.yaml (.../_core/bulk.yaml) +PASSED _core/info.yaml (.../_core/info.yaml) +SKIPPED indices/forcemerge.yaml (Skipped because distribution amazon-managed is not opensearch.org.) +``` ### Waiting for Tasks diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6d1916509..928ba4bad 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -21,6 +21,8 @@ properties: $ref: '#/definitions/Chapter' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' required: [chapters,description] additionalProperties: false @@ -85,6 +87,8 @@ definitions: $ref: '#/definitions/Output' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' retry: $ref: '#/definitions/Retry' required: [method, path] @@ -106,6 +110,13 @@ definitions: The semver range to execute the story or chapter against. type: string + Distributions: + description: | + The list of distributions that support this API. + type: array + items: + type: string + Retry: description: | Number of times to retry on error. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9aa5dbeaa..bc90411bb 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -9,6 +9,8 @@ paths: operationId: info.0 x-operation-group: info x-version-added: '1.0' + x-distributions-excluded: + - amazon-serverless description: Returns basic information about the cluster. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index ff0efdc21..cab8696c2 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -562,6 +562,14 @@ components: write_operations: description: The total number of write operations for the device completed since starting OpenSearch. type: number + read_time: + type: number + write_time: + type: number + queue_size: + type: number + io_time_in_millis: + $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' Jvm: type: object properties: @@ -1068,8 +1076,10 @@ components: type: boolean enforced: type: boolean + total_rejections_breakup: + $ref: '#/components/schemas/TotalRejectionsBreakup' total_rejections_breakup_shadow_mode: - $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode' + $ref: '#/components/schemas/TotalRejectionsBreakup' ShardSearchBackpressureStats: type: object properties: @@ -1140,7 +1150,11 @@ components: type: number cancellation_limit_reached_count: type: number - TotalRejectionsBreakupShadowMode: + cancelled_task_percentage: + type: number + current_cancellation_eligible_tasks_count: + type: number + TotalRejectionsBreakup: type: object properties: node_limits: diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7086c17ca..6fbcefdc2 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -102,14 +102,10 @@ components: search_pipelines: $ref: '#/components/schemas/NodeInfoSearchPipelines' required: - - attributes - build_hash - build_type - - host - - ip - name - roles - - transport_address - version NodeInfoHttp: type: object @@ -156,7 +152,7 @@ components: bundled_jdk: type: boolean using_bundled_jdk: - type: boolean + type: [boolean, 'null'] using_compressed_ordinary_object_pointers: oneOf: - type: boolean @@ -167,16 +163,9 @@ components: type: string required: - bundled_jdk - - gc_collectors - - input_arguments - mem - - memory_pools - pid - start_time_in_millis - - version - - vm_name - - vm_vendor - - vm_version NodeInfoJvmMemory: type: object properties: @@ -256,12 +245,8 @@ components: swap: $ref: '#/components/schemas/NodeInfoMemory' required: - - arch - available_processors - - name - - pretty_name - refresh_interval_in_millis - - version NodeInfoOSCPU: type: object properties: diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 67e245235..9cc644a6b 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -2,6 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org chapters: - synopsis: Get server info. path: / diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 8e7a04351..32f9b3c2d 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -124,4 +124,3 @@ chapters: title: Drive year: 2011 order: 2 - diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 38865e9e5..196cff4b3 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -53,7 +53,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat with master response (format=json). method: GET @@ -66,7 +65,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_master: 'true' - synopsis: Cat with cluster_manager response (format=json). @@ -80,7 +78,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_cluster_manager: 'true' - synopsis: Cat in different formats (format=yaml). @@ -93,9 +90,10 @@ chapters: content_type: application/yaml payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -105,9 +103,10 @@ chapters: content_type: application/cbor payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -117,5 +116,4 @@ chapters: content_type: application/smile payload: - node.total: '1' - status: yellow node.data: '1' diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index c20df6fe8..6b6130823 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -71,6 +71,8 @@ chapters: status: 200 content_type: application/yaml - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: @@ -79,6 +81,8 @@ chapters: status: 200 content_type: application/cbor - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index cb9dc328c..01dba1cae 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. + distributions: + - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index db4db9dd5..2e056a526 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 94087d494..0b1b0c784 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. +distributions: + - opensearch.org chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index d537e3a7c..cc1608cb6 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -2,6 +2,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index da87fec7b..0445ebd64 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -1,7 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. - +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 5bfada4b9..80906a7e3 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -11,6 +11,8 @@ epilogues: status: [200, 404] chapters: - synopsis: Get global settings. + distributions: + - opensearch.org path: /_settings method: GET parameters: @@ -23,6 +25,8 @@ chapters: response: status: 200 - synopsis: Get global settings (cluster_manager_timeout). + distributions: + - opensearch.org path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index a08efa0ce..8c036ae6e 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index 4bdac64e1..f92ee82d0 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml index f4f23e817..340a659d5 100644 --- a/tests/default/security/api/certificates.yaml +++ b/tests/default/security/api/certificates.yaml @@ -4,20 +4,18 @@ description: Test certificates endpoints. version: '> 2.14' # ADMIN-CERT only. These tests require explicit rest api admin privileges. -chapters: - - synopsis: Get node. - id: get_node - path: /_cat/nodes +prologues: + - path: /_cat/nodes + id: node method: GET parameters: - h: - - id full_id: true - response: - status: 200 - content_type: text/plain + size: 1 + format: json + h: id output: - node_id: payload + id: payload[0].id +chapters: - synopsis: Get all certificates. path: /_plugins/_security/api/certificates method: GET @@ -29,7 +27,7 @@ chapters: path: /_plugins/_security/api/certificates/{node_id} method: GET parameters: - node_id: ${get_node.node_id} + node_id: ${node.id} cert_type: all response: - status: 403 + status: 403 \ No newline at end of file diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index afbb7d5fa..314a73d45 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -73,6 +73,7 @@ export interface OpenSearchHttpClientOptions { export type OpenSearchHttpClientCliOptions = { opensearchUrl?: string + opensearchDistribution?: string, opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts index 52de7e899..836c802ea 100644 --- a/tools/src/_utils/semver.ts +++ b/tools/src/_utils/semver.ts @@ -9,7 +9,8 @@ import * as semver from 'semver' -export function coerce(version: string) : string { +export function coerce(version?: string): undefined | string { + if (version === undefined) return undefined return semver.coerce(version)?.toString() ?? version } diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index 06672285b..c463e49d9 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -16,7 +16,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemasValidator { diff --git a/tools/src/linter/components/OperationGroup.ts b/tools/src/linter/components/OperationGroup.ts index dc45765bc..f980e56b5 100644 --- a/tools/src/linter/components/OperationGroup.ts +++ b/tools/src/linter/components/OperationGroup.ts @@ -13,7 +13,7 @@ import ValidatorBase from './base/ValidatorBase' export default class OperationGroup extends ValidatorBase { static readonly OP_PRIORITY = ['operationId', 'x-operation-group', 'x-ignorable', 'deprecated', - 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', + 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', 'x-distributions-included', 'x-distributions-excluded', 'description', 'externalDocs', 'parameters', 'requestBody', 'responses'] name: string diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 3f2fb396b..a2ccbffa4 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -17,12 +17,14 @@ import * as semver from '../_utils/semver' export default class OpenApiVersionExtractor { private _spec?: Record private _source_spec: OpenAPIV3.Document - private _target_version: string + private _target_version?: string + private _target_distribution?: string private _logger: Logger - constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { + constructor(source_spec: OpenAPIV3.Document, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this._source_spec = source_spec this._target_version = semver.coerce(target_version) + this._target_distribution = target_distribution this._logger = logger this._spec = undefined } @@ -45,16 +47,37 @@ export default class OpenApiVersionExtractor { #extract() : void { this._logger.info(`Extracting version ${this._target_version} ...`) this.#remove_keys_not_matching_semver() + this.#remove_keys_not_matching_distribution() this.#remove_unused() } #exclude_per_semver(obj: any): boolean { + if (this._target_version == undefined) return false + const x_version_added = semver.coerce(obj['x-version-added'] as string) const x_version_removed = semver.coerce(obj['x-version-removed'] as string) - if (x_version_added && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { + if (x_version_added !== undefined && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { return true - } else if (x_version_removed && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + } else if (x_version_removed !== undefined && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + return true + } + + return false + } + + #exclude_per_distribution(obj: any): boolean { + if (this._target_distribution == undefined) return false + + const x_distributions_included = obj['x-distributions-included'] as string[] + + if (x_distributions_included?.length > 0 && !x_distributions_included.includes(this._target_distribution)) { + return true + } + + const x_distributions_excluded = obj['x-distributions-excluded'] as string[] + + if (x_distributions_excluded?.length > 0 && x_distributions_excluded.includes(this._target_distribution)) { return true } @@ -63,9 +86,16 @@ export default class OpenApiVersionExtractor { // Remove any elements that are x-version-added/removed incompatible with the target server version. #remove_keys_not_matching_semver(): void { + if (this._target_version == undefined) return delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this)) } + // Remove any elements that are x-distributions-included incompatible with the target distribution. + #remove_keys_not_matching_distribution(): void { + if (this._target_distribution === undefined) return + delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this)) + } + #remove_unused(): void { if (this._spec === undefined) return diff --git a/tools/src/tester/MergedOpenApiSpec.ts b/tools/src/tester/MergedOpenApiSpec.ts index ee21f55ac..d4a3e2604 100644 --- a/tools/src/tester/MergedOpenApiSpec.ts +++ b/tools/src/tester/MergedOpenApiSpec.ts @@ -20,21 +20,23 @@ export default class MergedOpenApiSpec { logger: Logger file_path: string target_version?: string + target_distribution?: string protected _spec: OpenAPIV3.Document | undefined - constructor (spec_path: string, target_version?: string, logger: Logger = new Logger()) { + constructor (spec_path: string, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this.logger = logger this.file_path = spec_path this.target_version = target_version + this.target_distribution = target_distribution } spec (): OpenAPIV3.Document { if (this._spec) return this._spec const merger = new OpenApiMerger(this.file_path, this.logger) var spec = merger.spec() - if (this.target_version !== undefined) { - const version_extractor = new OpenApiVersionExtractor(spec, this.target_version) + if (this.target_version !== undefined || this.target_distribution !== undefined) { + const version_extractor = new OpenApiVersionExtractor(spec, this.target_version, this.target_distribution) spec = version_extractor.extract() } const ctx = new SpecificationContext(this.file_path) diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 7f71e3870..01b55b5ad 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -19,7 +19,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemaValidator { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 9020968fa..3f396a3c6 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,8 +26,8 @@ export default class StoryEvaluator { this._supplemental_chapter_evaluator = supplemental_chapter_evaluator } - async evaluate({ story, display_path, full_path }: StoryFile, version?: string, dry_run: boolean = false): Promise { - if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { + if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { return { result: Result.SKIPPED, display_path, @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } + if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error } const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) return { display_path, @@ -76,7 +86,7 @@ export default class StoryEvaluator { } } - async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string): Promise { + async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { if (dry_run) { @@ -85,6 +95,9 @@ export default class StoryEvaluator { } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0b197d648..dc880f946 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -34,19 +34,25 @@ export default class TestRunner { this._result_logger = result_logger } - async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { + async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { - const info = await this._http_client.wait_until_available() - console.log(`OpenSearch ${ansi.green(info.version.number)}\n`) - version = info.version.number + if (distribution === 'amazon-serverless') { + // TODO: Fetch OpenSearch version when Amazon Serverless OpenSearch supports multiple. + version = '2.1' + } else { + const info = await this._http_client.wait_until_available() + version = info.version.number + } + + console.log(`OpenSearch ${ansi.green(version)}\n`) } for (const story_file of story_files) { - const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run) + const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 134888d7f..da025a34b 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -47,6 +47,9 @@ const command = new Command() .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(new Option('--opensearch-version ', 'target OpenSearch schema version').default(undefined)) + .addOption(new Option('--opensearch-distribution ', 'OpenSearch distribution') + .default('opensearch.org') + .env('OPENSEARCH_DISTRIBUTION')) .addOption(OPENSEARCH_URL_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) @@ -63,7 +66,7 @@ const command = new Command() const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) -const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) +const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, opts.opensearchDistribution, new Logger(LogLevel.error)) const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) @@ -73,7 +76,7 @@ const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapt const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) -runner.run(opts.testsPath, spec.api_version(), opts.dryRun) +runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( ({ results, failed }) => { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e418ff762..83d307b44 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -50,6 +50,14 @@ export type Payload = {} | any[] | string | number | boolean; * via the `definition` "Version". */ export type Version = string; +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export type Distributions = string[]; /** * Number of times to retry on error. * @@ -94,6 +102,7 @@ export interface Story { epilogues?: SupplementalChapter[]; chapters: Chapter[]; version?: Version; + distributions?: Distributions; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -112,6 +121,7 @@ export interface ChapterRequest { request?: Request; output?: Output; version?: Version; + distributions?: Distributions; retry?: Retry; } /** @@ -131,7 +141,7 @@ export interface Request { * The values are paths to the values in the response. * The values should be in the form: * - `payload.` for the payload - * - `headers.` for the headers + * - `headers.` for the headers. * * * This interface was referenced by `Story`'s JSON-Schema @@ -157,6 +167,9 @@ export interface ExpectedResponse { * via the `definition` "Warnings". */ export interface Warnings { + /** + * Enable/disable warnings about multiple paths being tested in the same story. + */ 'multiple-paths-detected'?: boolean; } /** diff --git a/tools/src/types.ts b/tools/src/types.ts index a9497dbcb..b6871d022 100644 --- a/tools/src/types.ts +++ b/tools/src/types.ts @@ -16,6 +16,8 @@ export interface OperationSpec extends OpenAPIV3.OperationObject { 'x-version-deprecated'?: string 'x-deprecation-message'?: string 'x-ignorable'?: boolean + 'x-distributions-included'?: string[] + 'x-distributions-excluded'?: string[] parameters?: OpenAPIV3.ReferenceObject[] requestBody?: OpenAPIV3.ReferenceObject diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index 61bc42622..ca3e14eb0 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -17,7 +17,7 @@ describe('extract() from a merged API spec', () => { const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') describe('1.3', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3') + const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') describe('write_to', () => { var temp: tmp.DirResult @@ -43,18 +43,18 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0', 'ignore') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless' ]) }) @@ -81,12 +81,12 @@ describe('extract() from a merged API spec', () => { }) describe('2.1', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'oracle-managed') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml index d0fa222f7..a4497ccbb 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml @@ -35,6 +35,10 @@ paths: x-version-removed: '2.0' added-1.3-removed-2.0: $ref: '#/components/responses/info@added-1.3-removed-2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -106,6 +110,8 @@ components: description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@removed-2.0: description: Removed in 2.0 via attribute next to ref. nodes.info@200: diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml index cfb4279d2..c684b7af9 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml @@ -57,6 +57,10 @@ paths: added-2.0: $ref: '#/components/responses/info@added-2.0' x-version-added: '2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -144,6 +148,8 @@ components: type: object info@added-2.0: description: Added in 2.0 via attribute next to ref. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. nodes.info@200: description: All nodes. content: diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 7e6c64641..db852a030 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -13,7 +13,7 @@ import MergedOpenApiSpec from "tester/MergedOpenApiSpec" describe('merged API spec', () => { describe('defaults', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, undefined, new Logger()) test('has an api version', () => { expect(spec.api_version()).toEqual('1.2.3') @@ -30,7 +30,8 @@ describe('merged API spec', () => { test('has all responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1' + '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) @@ -65,31 +66,67 @@ describe('merged API spec', () => { }) describe('1.3', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'oracle-managed', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', undefined, new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('2.0 amazon-serverless', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'amazon-serverless', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all' + ]) + }) + }) + + describe('2.0 oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'oracle-managed', new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.1', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml new file mode 100644 index 000000000..cc4ef9a19 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml + +result: SKIPPED +description: This story should be skipped because of distributions. +message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index 68337dc67..0c0a09234 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -28,6 +28,20 @@ paths: $ref: '#/components/responses/info@500' '503': $ref: '#/components/responses/info@503' + distributed-included-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions-included: + - amazon-managed + - amazon-serverless + - opensearch.org + distributed-included-amazon-managed: + $ref: '#/components/responses/info@distributed-amazon-managed' + x-distributions-included: + - amazon-managed + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless components: responses: info@200: @@ -76,6 +90,10 @@ components: info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' + info@distributed-amazon-managed: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@500: content: application/json: diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml new file mode 100644 index 000000000..8efebbb2f --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml @@ -0,0 +1,9 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions. +distributions: + - another + - some +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index ce11b6ad8..d6d7e5639 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index b0333504f..0259c29ac 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -16,7 +16,7 @@ test('stories folder', async () => { const info = await opensearch_http_client.wait_until_available() expect(info.version).toBeDefined() - const run = await test_runner.run('tools/tests/tester/fixtures/stories') + const run = await test_runner.run('tools/tests/tester/fixtures/stories', undefined, 'opensearch.org') expect(run.failed).toBeTruthy() @@ -29,14 +29,15 @@ test('stories folder', async () => { } const passed = load_expected_evaluation('passed', true) - const skipped = load_expected_evaluation('skipped/semver', true) + const skipped_semver = load_expected_evaluation('skipped/semver', true) + const skipped_distributions = load_expected_evaluation('skipped/distributions', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const output_error = load_expected_evaluation('error/output_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped] + const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] expect(actual_evaluations).toEqual(expected_evaluations) })