From e865cf6755dc551e96fbd2f4252fc50a869db01a Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:42:43 -0500 Subject: [PATCH 01/21] Add Segments API test (#477) * Add Segments API test Signed-off-by: Archer * Fix errors Signed-off-by: Archer * Update changelog Signed-off-by: Archer --------- Signed-off-by: Archer --- tests/default/indices/segments.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tests/default/indices/segments.yaml diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml new file mode 100644 index 000000000..da87fec7b --- /dev/null +++ b/tests/default/indices/segments.yaml @@ -0,0 +1,28 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: This story tests the Segments API. + +prologues: + - path: /movies + method: PUT +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get details about Lucene segments. + path: /_segments + method: GET + parameters: + expand_wildcards: none + response: + status: 200 + - synopsis: Get details about Lucene segments inside the specified index. + path: /{index}/_segments + method: GET + parameters: + index: movies + expand_wildcards: all + allow_no_indices: false + response: + status: 200 From 18587f56d95de28f0229a4fe98dfd6d15b850f62 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 12:59:06 -0400 Subject: [PATCH 02/21] Added support for SigV4. (#476) * Added support for running tests against Amazon OpenSearch. Signed-off-by: dblock --- .cspell | 5 +- CHANGELOG.md | 1 + TESTING_GUIDE.md | 30 +- eslint.config.mjs | 6 +- package-lock.json | 2251 ++++++++++++++++- package.json | 2 + tests/default/cat/plugins.yaml | 2 - tools/src/OpenSearchHttpClient.ts | 116 +- .../dump-cluster-spec/dump-cluster-spec.ts | 2 +- tools/src/tester/test.ts | 12 +- .../tests/tester/OpenSearchHttpClient.test.ts | 66 + tools/tests/tester/helpers.ts | 6 +- 12 files changed, 2392 insertions(+), 107 deletions(-) create mode 100644 tools/tests/tester/OpenSearchHttpClient.test.ts diff --git a/.cspell b/.cspell index 080c3937b..1542876a8 100644 --- a/.cspell +++ b/.cspell @@ -2,6 +2,7 @@ aarch actiongroup actiongroups aggregatable +aoss APIV argjson asciifolding @@ -129,6 +130,7 @@ readingform rebalance Rebalance recoverysource +Refn reindex Reindex relo @@ -183,5 +185,4 @@ urldecode vectory whoamiprotected wordnet -Yrtsd -Refn +Yrtsd \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d0e0e18a2..f5590437d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) +- Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) ### Changed diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index d50d4f5b8..5c066549e 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -1,9 +1,11 @@ - [Spec Testing Guide](#spec-testing-guide) - - [Running Spec Tests Locally](#running-spec-tests-locally) - - [Common Errors](#common-errors) - - [401 Unauthorized](#401-unauthorized) - - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + - [Running Spec Tests](#running-spec-tests) + - [Running Spec Tests Locally](#running-spec-tests-locally) + - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) + - [Common Errors](#common-errors) + - [401 Unauthorized](#401-unauthorized) + - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) - [Writing Spec Tests](#writing-spec-tests) - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) @@ -18,7 +20,9 @@ We have devised our own test framework to test the spec against an OpenSearch cluster. We're still adding more features to the framework as the needs arise, and this document will be updated accordingly. This test framework has also been integrated into the repo's CI/CD pipeline. Checkout the [test-spec](.github/workflows/test-spec.yml) workflow for more details. -## Running Spec Tests Locally +## Running Spec Tests + +### Running Spec Tests Locally Set up an OpenSearch cluster with Docker: @@ -44,11 +48,25 @@ Verbose output: npm run test:spec -- --opensearch-insecure --verbose ``` +### Running Spec Tests with Amazon OpenSearch + +Use an Amazon OpenSearch service instance. + +```bash +export AWS_ACCESS_KEY_ID=<> +export AWS_SECRET_ACCESS_KEY=<> +export AWS_SESSION_TOKEN=<> +export AWS_REGION=us-west-2 +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec +``` + ### Common Errors #### 401 Unauthorized -Remember to set the `OPENSEARCH_PASSWORD` environment variable everytime you start a new shell to run the tests. +Remember to set the `OPENSEARCH_PASSWORD` or `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables every time you start a new shell to run the tests. #### FORBIDDEN/10/cluster create-index blocked (api) diff --git a/eslint.config.mjs b/eslint.config.mjs index 337f14721..f782d0a2b 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -8,6 +8,7 @@ import pluginTs from '@typescript-eslint/eslint-plugin' import pluginYml from 'eslint-plugin-yml' import pluginCspell from '@cspell/eslint-plugin' import pluginStylistic from '@stylistic/eslint-plugin' +import pluginJest from 'eslint-plugin-jest' export default [ pluginJs.configs.recommended, @@ -28,7 +29,8 @@ export default [ 'license-header': pluginLicenseHeader, 'eslint-comments': pluginComments, '@cspell': pluginCspell, - '@stylistic': pluginStylistic + '@stylistic': pluginStylistic, + 'jest': pluginJest }, rules: { ...pluginJs.configs.recommended.rules, @@ -78,6 +80,8 @@ export default [ ], '@typescript-eslint/require-await': 'error', '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }], + '@typescript-eslint/unbound-method': 'off', + 'jest/unbound-method': 'error', 'array-callback-return': 'off', 'indent': ['error', 2, { 'SwitchCase': 1 }], 'new-cap': 'off', diff --git a/package-lock.json b/package-lock.json index f86e80d30..e1156510e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", + "aws4-axios": "^3.3.7", "axios": "^1.7.1", "cbor": "^9.0.2", "commander": "^12.0.0", @@ -33,6 +34,7 @@ "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", "eslint-plugin-import": "^2.29.1", + "eslint-plugin-jest": "^28.8.0", "eslint-plugin-license-header": "^0.6.1", "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", @@ -104,6 +106,578 @@ "openapi-types": ">=7" } }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.624.0.tgz", + "integrity": "sha512-EX6EF+rJzMPC5dcdsu40xSi2To7GSvdGQNIpe97pD9WvZwM9tRNQnNM4T6HA4gjV1L6Jwk8rBlG/CnveXtLEMw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.624.0.tgz", + "integrity": "sha512-Ki2uKYJKKtfHxxZsiMTOvJoVRP6b2pZ1u3rcUb2m/nVgBPUfLdl8ZkGpqE29I+t5/QaS/sEdbn6cgMUZwl+3Dg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.624.0" + } + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.624.0.tgz", + "integrity": "sha512-k36fLZCb2nfoV/DKK3jbRgO/Yf7/R80pgYfMiotkGjnZwDmRvNN08z4l06L9C+CieazzkgRxNUzyppsYcYsQaw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.624.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.624.0.tgz", + "integrity": "sha512-WyFmPbhRIvtWi7hBp8uSFy+iPpj8ccNV/eX86hwF4irMjfc/FtsGVIAeBXxXM/vGCjkdfEzOnl+tJ2XACD4OXg==", + "dependencies": { + "@smithy/core": "^2.3.2", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.622.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.622.0.tgz", + "integrity": "sha512-VUHbr24Oll1RK3WR8XLUugLpgK9ZuxEm/NVeVqyFts1Ck9gsKpRg1x4eH7L7tW3SJ4TDEQNMbD7/7J+eoL2svg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.624.0.tgz", + "integrity": "sha512-mMoNIy7MO2WTBbdqMyLpbt6SZpthE6e0GkRYpsd0yozPt0RZopcBhEh+HG1U9Y1PVODo+jcMk353vAi61CfnhQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.624.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.624.0.tgz", + "integrity": "sha512-vYyGK7oNpd81BdbH5IlmQ6zfaQqU+rPwsKTDDBeLRjshtrGXOEpfoahVpG9PX0ibu32IOWp4ZyXBNyVrnvcMOw==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-ini": "3.624.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.624.0.tgz", + "integrity": "sha512-A02bayIjU9APEPKr3HudrFHEx0WfghoSPsPopckDkW7VBqO4wizzcxr75Q9A3vNX+cwg0wCN6UitTNe6pVlRaQ==", + "dependencies": { + "@aws-sdk/client-sso": "3.624.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.621.0" + } + }, + "node_modules/@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.620.0.tgz", + "integrity": "sha512-bvS6etn+KsuL32ubY5D3xNof1qkenpbJXf/ugGXbg0n98DvDFQ/F+SMLxHgbnER5dsKYchNnhmtI6/FC3HFu/A==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.614.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", + "dependencies": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, "node_modules/@babel/code-frame": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.6.tgz", @@ -1767,50 +2341,562 @@ "fastq": "^1.6.0" }, "engines": { - "node": ">= 8" + "node": ">= 8" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@pkgr/core": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", + "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", + "engines": { + "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/unts" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "peer": true, + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "peer": true, + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.3.2.tgz", + "integrity": "sha512-in5wwt6chDBcUv1Lw1+QzZxN9fBffi+qOixfb65yK4sDuKG7zAUO9HAFqmVzsZM3N+3tTyvZjtnDXePpvp007Q==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "dependencies": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "dependencies": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-retry": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.14.tgz", + "integrity": "sha512-7ZaWZJOjUxa5hgmuMspyt8v/zVsh0GXYuF7OvCmdcbVa/xbnKQoYC+uYKunAqRGTkxjOyuOCw9rmFUFOqqC0eQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "dependencies": { + "@smithy/types": "^3.3.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/smithy-client": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.12.tgz", + "integrity": "sha512-wtm8JtsycthkHy1YA4zjIh2thJgIQ9vGkoR639DBx5lLlLNU0v4GARpQZkr2WjXue74nZ7MiTSWfVrLkyD8RkA==", + "dependencies": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.14.tgz", + "integrity": "sha512-0iwTgKKmAIf+vFLV8fji21Jb2px11ktKVxbX6LIDPAUJyWQqGqBVfwba7xwa1f2FZUoolYQgLvxQEpJycXuQ5w==", + "dependencies": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.14.tgz", + "integrity": "sha512-e9uQarJKfXApkTMMruIdxHprhcXivH1flYCe8JRDTzkkLx8dA3V5J8GZlST9yfDiRWkJpZJlUXGN9Rc9Ade3OQ==", + "dependencies": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "dependencies": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, + "node_modules/@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + }, "engines": { - "node": ">=14" + "node": ">=16.0.0" } }, - "node_modules/@pkgr/core": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@pkgr/core/-/core-0.1.1.tgz", - "integrity": "sha512-cq8o4cWH0ibXh9VGi5P20Tu9XF/0fFXl9EUinr9QfTM7a7p0oTA4iJRCQWppXR1Pg8dSM0UCItCkPwsk9qWWYA==", - "engines": { - "node": "^12.20.0 || ^14.18.0 || >=16.0.0" + "node_modules/@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "dependencies": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" }, - "funding": { - "url": "https://opencollective.com/unts" + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@sinclair/typebox": { - "version": "0.27.8", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", - "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "peer": true, + "node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", "dependencies": { - "type-detect": "4.0.8" + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "peer": true, + "node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", "dependencies": { - "@sinonjs/commons": "^3.0.0" + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@stylistic/eslint-plugin": { @@ -2819,6 +3905,26 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws4": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz", + "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==" + }, + "node_modules/aws4-axios": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/aws4-axios/-/aws4-axios-3.3.7.tgz", + "integrity": "sha512-TVuUKe6SNv5vqrE6VLSR+eN8JJhijA28qKdMGKbY23TcksYl0KGrW9dADWRwLUiLTrKq/rTucjaoJzNruNf4RQ==", + "dependencies": { + "@aws-sdk/client-sts": "^3.4.1", + "aws4": "^1.12.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "axios": ">=1.6.0" + } + }, "node_modules/axios": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", @@ -2955,6 +4061,11 @@ "resolved": "https://registry.npmjs.org/bit-buffer/-/bit-buffer-0.2.5.tgz", "integrity": "sha512-x1yGnmXvFg6e3DiyRztElbcn1bsCTFSoM/ncAzY62uE0JdTl5xlKJd0ooqLYoPbhdsnpehSIQrdIvclcZJYwiA==" }, + "node_modules/bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -4170,6 +5281,30 @@ "semver": "bin/semver.js" } }, + "node_modules/eslint-plugin-jest": { + "version": "28.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-28.8.0.tgz", + "integrity": "sha512-Tubj1hooFxCl52G4qQu0edzV/+EZzPUeN8p2NnW5uu4fbDs+Yo7+qDVDc4/oG3FbCqEBmu/OC3LSsyiU22oghw==", + "dependencies": { + "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "engines": { + "node": "^16.10.0 || ^18.12.0 || >=20.0.0" + }, + "peerDependencies": { + "@typescript-eslint/eslint-plugin": "^6.0.0 || ^7.0.0 || ^8.0.0", + "eslint": "^7.0.0 || ^8.0.0 || ^9.0.0", + "jest": "*" + }, + "peerDependenciesMeta": { + "@typescript-eslint/eslint-plugin": { + "optional": true + }, + "jest": { + "optional": true + } + } + }, "node_modules/eslint-plugin-license-header": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/eslint-plugin-license-header/-/eslint-plugin-license-header-0.6.1.tgz", @@ -4602,6 +5737,27 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, + "node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -7611,6 +8767,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -8049,6 +9210,18 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", @@ -8289,61 +9462,533 @@ "node": ">=6" } }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + }, + "dependencies": { + "@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "peer": true, + "requires": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "@apidevtools/json-schema-ref-parser": { + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", + "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "requires": { + "@jsdevtools/ono": "^7.1.3", + "call-me-maybe": "^1.0.1", + "js-yaml": "^3.13.1" + } + }, + "@apidevtools/openapi-schemas": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", + "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==" + }, + "@apidevtools/swagger-methods": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", + "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + }, + "@apidevtools/swagger-parser": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz", + "integrity": "sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==", + "requires": { + "@apidevtools/json-schema-ref-parser": "9.0.6", + "@apidevtools/openapi-schemas": "^2.1.0", + "@apidevtools/swagger-methods": "^3.0.2", + "@jsdevtools/ono": "^7.1.3", + "ajv": "^8.6.3", + "ajv-draft-04": "^1.0.0", + "call-me-maybe": "^1.0.1" + } + }, + "@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "requires": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + } + } + }, + "@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "requires": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + } + }, + "@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "requires": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + }, + "dependencies": { + "@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "requires": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "requires": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + } + } + } + }, + "@aws-sdk/client-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.624.0.tgz", + "integrity": "sha512-EX6EF+rJzMPC5dcdsu40xSi2To7GSvdGQNIpe97pD9WvZwM9tRNQnNM4T6HA4gjV1L6Jwk8rBlG/CnveXtLEMw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/client-sso-oidc": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.624.0.tgz", + "integrity": "sha512-Ki2uKYJKKtfHxxZsiMTOvJoVRP6b2pZ1u3rcUb2m/nVgBPUfLdl8ZkGpqE29I+t5/QaS/sEdbn6cgMUZwl+3Dg==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/client-sts": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.624.0.tgz", + "integrity": "sha512-k36fLZCb2nfoV/DKK3jbRgO/Yf7/R80pgYfMiotkGjnZwDmRvNN08z4l06L9C+CieazzkgRxNUzyppsYcYsQaw==", + "requires": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.624.0", + "@aws-sdk/core": "3.624.0", + "@aws-sdk/credential-provider-node": "3.624.0", + "@aws-sdk/middleware-host-header": "3.620.0", + "@aws-sdk/middleware-logger": "3.609.0", + "@aws-sdk/middleware-recursion-detection": "3.620.0", + "@aws-sdk/middleware-user-agent": "3.620.0", + "@aws-sdk/region-config-resolver": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@aws-sdk/util-user-agent-browser": "3.609.0", + "@aws-sdk/util-user-agent-node": "3.614.0", + "@smithy/config-resolver": "^3.0.5", + "@smithy/core": "^2.3.2", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/hash-node": "^3.0.3", + "@smithy/invalid-dependency": "^3.0.3", + "@smithy/middleware-content-length": "^3.0.5", + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.14", + "@smithy/util-defaults-mode-node": "^3.0.14", + "@smithy/util-endpoints": "^2.0.5", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/core": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.624.0.tgz", + "integrity": "sha512-WyFmPbhRIvtWi7hBp8uSFy+iPpj8ccNV/eX86hwF4irMjfc/FtsGVIAeBXxXM/vGCjkdfEzOnl+tJ2XACD4OXg==", + "requires": { + "@smithy/core": "^2.3.2", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/signature-v4": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-env": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.620.1.tgz", + "integrity": "sha512-ExuILJ2qLW5ZO+rgkNRj0xiAipKT16Rk77buvPP8csR7kkCflT/gXTyzRe/uzIiETTxM7tr8xuO9MP/DQXqkfg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-http": { + "version": "3.622.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.622.0.tgz", + "integrity": "sha512-VUHbr24Oll1RK3WR8XLUugLpgK9ZuxEm/NVeVqyFts1Ck9gsKpRg1x4eH7L7tW3SJ4TDEQNMbD7/7J+eoL2svg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-ini": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.624.0.tgz", + "integrity": "sha512-mMoNIy7MO2WTBbdqMyLpbt6SZpthE6e0GkRYpsd0yozPt0RZopcBhEh+HG1U9Y1PVODo+jcMk353vAi61CfnhQ==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-node": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.624.0.tgz", + "integrity": "sha512-vYyGK7oNpd81BdbH5IlmQ6zfaQqU+rPwsKTDDBeLRjshtrGXOEpfoahVpG9PX0ibu32IOWp4ZyXBNyVrnvcMOw==", + "requires": { + "@aws-sdk/credential-provider-env": "3.620.1", + "@aws-sdk/credential-provider-http": "3.622.0", + "@aws-sdk/credential-provider-ini": "3.624.0", + "@aws-sdk/credential-provider-process": "3.620.1", + "@aws-sdk/credential-provider-sso": "3.624.0", + "@aws-sdk/credential-provider-web-identity": "3.621.0", + "@aws-sdk/types": "3.609.0", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-process": { + "version": "3.620.1", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.620.1.tgz", + "integrity": "sha512-hWqFMidqLAkaV9G460+1at6qa9vySbjQKKc04p59OT7lZ5cO5VH5S4aI05e+m4j364MBROjjk2ugNvfNf/8ILg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-sso": { + "version": "3.624.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.624.0.tgz", + "integrity": "sha512-A02bayIjU9APEPKr3HudrFHEx0WfghoSPsPopckDkW7VBqO4wizzcxr75Q9A3vNX+cwg0wCN6UitTNe6pVlRaQ==", + "requires": { + "@aws-sdk/client-sso": "3.624.0", + "@aws-sdk/token-providers": "3.614.0", + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/credential-provider-web-identity": { + "version": "3.621.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.621.0.tgz", + "integrity": "sha512-w7ASSyfNvcx7+bYGep3VBgC3K6vEdLmlpjT7nSIHxxQf+WSdvy+HynwJosrpZax0sK5q0D1Jpn/5q+r5lwwW6w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-host-header": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.620.0.tgz", + "integrity": "sha512-VMtPEZwqYrII/oUkffYsNWY9PZ9xpNJpMgmyU0rlDQ25O1c0Hk3fJmZRe6pEkAJ0omD7kLrqGl1DUjQVxpd/Rg==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-logger": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.609.0.tgz", + "integrity": "sha512-S62U2dy4jMDhDFDK5gZ4VxFdWzCtLzwbYyFZx2uvPYTECkepLUfzLic2BHg2Qvtu4QjX+oGE3P/7fwaGIsGNuQ==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-recursion-detection": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.620.0.tgz", + "integrity": "sha512-nh91S7aGK3e/o1ck64sA/CyoFw+gAYj2BDOnoNa6ouyCrVJED96ZXWbhye/fz9SgmNUZR2g7GdVpiLpMKZoI5w==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/middleware-user-agent": { + "version": "3.620.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.620.0.tgz", + "integrity": "sha512-bvS6etn+KsuL32ubY5D3xNof1qkenpbJXf/ugGXbg0n98DvDFQ/F+SMLxHgbnER5dsKYchNnhmtI6/FC3HFu/A==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@aws-sdk/util-endpoints": "3.614.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/region-config-resolver": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.614.0.tgz", + "integrity": "sha512-vDCeMXvic/LU0KFIUjpC3RiSTIkkvESsEfbVHiHH0YINfl8HnEqR5rj+L8+phsCeVg2+LmYwYxd5NRz4PHxt5g==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@aws-sdk/token-providers": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.614.0.tgz", + "integrity": "sha512-okItqyY6L9IHdxqs+Z116y5/nda7rHxLvROxtAJdLavWTYDydxrZstImNgGWTeVdmc0xX2gJCI77UYUTQWnhRw==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } - } - }, - "dependencies": { - "@ampproject/remapping": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", - "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", - "peer": true, + }, + "@aws-sdk/types": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.609.0.tgz", + "integrity": "sha512-+Tqnh9w0h2LcrUsdXyT1F8mNhXz+tVYBtP19LpeEGntmvHwa2XzvLUCWpoIAIVsHp5+HdB2X9Sn0KAtmbFXc2Q==", "requires": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } }, - "@apidevtools/json-schema-ref-parser": { - "version": "9.0.6", - "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.0.6.tgz", - "integrity": "sha512-M3YgsLjI0lZxvrpeGVk9Ap032W6TPQkH6pRAZz81Ac3WUNF79VQooAFnp8umjvVzUmD93NkogxEwbSce7qMsUg==", + "@aws-sdk/util-endpoints": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.614.0.tgz", + "integrity": "sha512-wK2cdrXHH4oz4IomV/yrGkftU9A+ITB6nFL+rxxyO78is2ifHJpFdV4aqk4LSkXYPi6CXWNru/Dqc7yiKXgJPw==", "requires": { - "@jsdevtools/ono": "^7.1.3", - "call-me-maybe": "^1.0.1", - "js-yaml": "^3.13.1" + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "@smithy/util-endpoints": "^2.0.5", + "tslib": "^2.6.2" } }, - "@apidevtools/openapi-schemas": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@apidevtools/openapi-schemas/-/openapi-schemas-2.1.0.tgz", - "integrity": "sha512-Zc1AlqrJlX3SlpupFGpiLi2EbteyP7fXmUOGup6/DnkRgjP9bgMM/ag+n91rsv0U1Gpz0H3VILA/o3bW7Ua6BQ==" + "@aws-sdk/util-locate-window": { + "version": "3.568.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.568.0.tgz", + "integrity": "sha512-3nh4TINkXYr+H41QaPelCceEB2FXP3fxp93YZXB/kqJvX0U9j0N0Uk45gvsjmEPzG8XxkPEeLIfT2I1M7A6Lig==", + "requires": { + "tslib": "^2.6.2" + } }, - "@apidevtools/swagger-methods": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@apidevtools/swagger-methods/-/swagger-methods-3.0.2.tgz", - "integrity": "sha512-QAkD5kK2b1WfjDS/UQn/qQkbwF31uqRjPTrsCs5ZG9BQGAkjwvqGFjjPqAuzac/IYzpPtRzjCP1WrTuAIjMrXg==" + "@aws-sdk/util-user-agent-browser": { + "version": "3.609.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.609.0.tgz", + "integrity": "sha512-fojPU+mNahzQ0YHYBsx0ZIhmMA96H+ZIZ665ObU9tl+SGdbLneVZVikGve+NmHTQwHzwkFsZYYnVKAkreJLAtA==", + "requires": { + "@aws-sdk/types": "3.609.0", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } }, - "@apidevtools/swagger-parser": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/@apidevtools/swagger-parser/-/swagger-parser-10.1.0.tgz", - "integrity": "sha512-9Kt7EuS/7WbMAUv2gSziqjvxwDbFSg3Xeyfuj5laUODX8o/k/CpsAKiQ8W7/R88eXFTMbJYg6+7uAmOWNKmwnw==", + "@aws-sdk/util-user-agent-node": { + "version": "3.614.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.614.0.tgz", + "integrity": "sha512-15ElZT88peoHnq5TEoEtZwoXTXRxNrk60TZNdpl/TUBJ5oNJ9Dqb5Z4ryb8ofN6nm9aFf59GVAerFDz8iUoHBA==", "requires": { - "@apidevtools/json-schema-ref-parser": "9.0.6", - "@apidevtools/openapi-schemas": "^2.1.0", - "@apidevtools/swagger-methods": "^3.0.2", - "@jsdevtools/ono": "^7.1.3", - "ajv": "^8.6.3", - "ajv-draft-04": "^1.0.0", - "call-me-maybe": "^1.0.1" + "@aws-sdk/types": "3.609.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" } }, "@babel/code-frame": { @@ -9702,6 +11347,413 @@ "@sinonjs/commons": "^3.0.0" } }, + "@smithy/abort-controller": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.1.tgz", + "integrity": "sha512-MBJBiidoe+0cTFhyxT8g+9g7CeVccLM0IOKKUMCNQ1CNMJ/eIfoo0RTfVrXOONEI1UCN1W+zkiHSbzUNE9dZtQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/config-resolver": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.5.tgz", + "integrity": "sha512-SkW5LxfkSI1bUC74OtfBbdz+grQXYiPYolyu8VfpLIjEoN/sHVBlLeGXMQ1vX4ejkgfv6sxVbQJ32yF2cl1veA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/core": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.3.2.tgz", + "integrity": "sha512-in5wwt6chDBcUv1Lw1+QzZxN9fBffi+qOixfb65yK4sDuKG7zAUO9HAFqmVzsZM3N+3tTyvZjtnDXePpvp007Q==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-retry": "^3.0.14", + "@smithy/middleware-serde": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/credential-provider-imds": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.0.tgz", + "integrity": "sha512-0SCIzgd8LYZ9EJxUjLXBmEKSZR/P/w6l7Rz/pab9culE/RWuqelAKGJvn5qUOl8BgX8Yj5HWM50A5hiB/RzsgA==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/fetch-http-handler": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.2.4.tgz", + "integrity": "sha512-kBprh5Gs5h7ug4nBWZi1FZthdqSM+T7zMmsZxx0IBvWUn7dK3diz2SHn7Bs4dQGFDk8plDv375gzenDoNwrXjg==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/hash-node": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.3.tgz", + "integrity": "sha512-2ctBXpPMG+B3BtWSGNnKELJ7SH9e4TNefJS0cd2eSkOOROeBnnVBnAy9LtJ8tY4vUEoe55N4CNPxzbWvR39iBw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/invalid-dependency": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.3.tgz", + "integrity": "sha512-ID1eL/zpDULmHJbflb864k72/SNOZCADRc9i7Exq3RUNJw6raWUSlFEQ+3PX3EYs++bTxZB2dE9mEHTQLv61tw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-content-length": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.5.tgz", + "integrity": "sha512-ILEzC2eyxx6ncej3zZSwMpB5RJ0zuqH7eMptxC4KN3f+v9bqT8ohssKbhNR78k/2tWW+KS5Spw+tbPF4Ejyqvw==", + "requires": { + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-endpoint": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.1.0.tgz", + "integrity": "sha512-5y5aiKCEwg9TDPB4yFE7H6tYvGFf1OJHNczeY10/EFF8Ir8jZbNntQJxMWNfeQjC1mxPsaQ6mR9cvQbf+0YeMw==", + "requires": { + "@smithy/middleware-serde": "^3.0.3", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/url-parser": "^3.0.3", + "@smithy/util-middleware": "^3.0.3", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-retry": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.14.tgz", + "integrity": "sha512-7ZaWZJOjUxa5hgmuMspyt8v/zVsh0GXYuF7OvCmdcbVa/xbnKQoYC+uYKunAqRGTkxjOyuOCw9rmFUFOqqC0eQ==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/protocol-http": "^4.1.0", + "@smithy/service-error-classification": "^3.0.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-retry": "^3.0.3", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + } + }, + "@smithy/middleware-serde": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.3.tgz", + "integrity": "sha512-puUbyJQBcg9eSErFXjKNiGILJGtiqmuuNKEYNYfUD57fUl4i9+mfmThtQhvFXU0hCVG0iEJhvQUipUf+/SsFdA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/middleware-stack": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.3.tgz", + "integrity": "sha512-r4klY9nFudB0r9UdSMaGSyjyQK5adUyPnQN/ZM6M75phTxOdnc/AhpvGD1fQUvgmqjQEBGCwpnPbDm8pH5PapA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-config-provider": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.4.tgz", + "integrity": "sha512-YvnElQy8HR4vDcAjoy7Xkx9YT8xZP4cBXcbJSgm/kxmiQu08DwUwj8rkGnyoJTpfl/3xYHH+d8zE+eHqoDCSdQ==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/shared-ini-file-loader": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/node-http-handler": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.1.4.tgz", + "integrity": "sha512-+UmxgixgOr/yLsUxcEKGH0fMNVteJFGkmRltYFHnBMlogyFdpzn2CwqWmxOrfJELhV34v0WSlaqG1UtE1uXlJg==", + "requires": { + "@smithy/abort-controller": "^3.1.1", + "@smithy/protocol-http": "^4.1.0", + "@smithy/querystring-builder": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/property-provider": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.3.tgz", + "integrity": "sha512-zahyOVR9Q4PEoguJ/NrFP4O7SMAfYO1HLhB18M+q+Z4KFd4V2obiMnlVoUFzFLSPeVt1POyNWneHHrZaTMoc/g==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/protocol-http": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.0.tgz", + "integrity": "sha512-dPVoHYQ2wcHooGXg3LQisa1hH0e4y0pAddPMeeUPipI1tEOqL6A4N0/G7abeq+K8wrwSgjk4C0wnD1XZpJm5aA==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-builder": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.3.tgz", + "integrity": "sha512-vyWckeUeesFKzCDaRwWLUA1Xym9McaA6XpFfAK5qI9DKJ4M33ooQGqvM4J+LalH4u/Dq9nFiC8U6Qn1qi0+9zw==", + "requires": { + "@smithy/types": "^3.3.0", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/querystring-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.3.tgz", + "integrity": "sha512-zahM1lQv2YjmznnfQsWbYojFe55l0SLG/988brlLv1i8z3dubloLF+75ATRsqPBboUXsW6I9CPGE5rQgLfY0vQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/service-error-classification": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.3.tgz", + "integrity": "sha512-Jn39sSl8cim/VlkLsUhRFq/dKDnRUFlfRkvhOJaUbLBXUsLRLNf9WaxDv/z9BjuQ3A6k/qE8af1lsqcwm7+DaQ==", + "requires": { + "@smithy/types": "^3.3.0" + } + }, + "@smithy/shared-ini-file-loader": { + "version": "3.1.4", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.4.tgz", + "integrity": "sha512-qMxS4hBGB8FY2GQqshcRUy1K6k8aBWP5vwm8qKkCT3A9K2dawUwOIJfqh9Yste/Bl0J2lzosVyrXDj68kLcHXQ==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/signature-v4": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.1.0.tgz", + "integrity": "sha512-aRryp2XNZeRcOtuJoxjydO6QTaVhxx/vjaR+gx7ZjaFgrgPRyZ3HCTbfwqYj6ZWEBHkCSUfcaymKPURaByukag==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.3", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/smithy-client": { + "version": "3.1.12", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.12.tgz", + "integrity": "sha512-wtm8JtsycthkHy1YA4zjIh2thJgIQ9vGkoR639DBx5lLlLNU0v4GARpQZkr2WjXue74nZ7MiTSWfVrLkyD8RkA==", + "requires": { + "@smithy/middleware-endpoint": "^3.1.0", + "@smithy/middleware-stack": "^3.0.3", + "@smithy/protocol-http": "^4.1.0", + "@smithy/types": "^3.3.0", + "@smithy/util-stream": "^3.1.3", + "tslib": "^2.6.2" + } + }, + "@smithy/types": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.3.0.tgz", + "integrity": "sha512-IxvBBCTFDHbVoK7zIxqA1ZOdc4QfM5HM7rGleCuHi7L1wnKv5Pn69xXJQ9hgxH60ZVygH9/JG0jRgtUncE3QUA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/url-parser": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.3.tgz", + "integrity": "sha512-pw3VtZtX2rg+s6HMs6/+u9+hu6oY6U7IohGhVNnjbgKy86wcIsSZwgHrFR+t67Uyxvp4Xz3p3kGXXIpTNisq8A==", + "requires": { + "@smithy/querystring-parser": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "requires": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-browser": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.14.tgz", + "integrity": "sha512-0iwTgKKmAIf+vFLV8fji21Jb2px11ktKVxbX6LIDPAUJyWQqGqBVfwba7xwa1f2FZUoolYQgLvxQEpJycXuQ5w==", + "requires": { + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-defaults-mode-node": { + "version": "3.0.14", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.14.tgz", + "integrity": "sha512-e9uQarJKfXApkTMMruIdxHprhcXivH1flYCe8JRDTzkkLx8dA3V5J8GZlST9yfDiRWkJpZJlUXGN9Rc9Ade3OQ==", + "requires": { + "@smithy/config-resolver": "^3.0.5", + "@smithy/credential-provider-imds": "^3.2.0", + "@smithy/node-config-provider": "^3.1.4", + "@smithy/property-provider": "^3.1.3", + "@smithy/smithy-client": "^3.1.12", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-endpoints": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.5.tgz", + "integrity": "sha512-ReQP0BWihIE68OAblC/WQmDD40Gx+QY1Ez8mTdFMXpmjfxSyz2fVQu3A4zXRfQU9sZXtewk3GmhfOHswvX+eNg==", + "requires": { + "@smithy/node-config-provider": "^3.1.4", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-middleware": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.3.tgz", + "integrity": "sha512-l+StyYYK/eO3DlVPbU+4Bi06Jjal+PFLSMmlWM1BEwyLxZ3aKkf1ROnoIakfaA7mC6uw3ny7JBkau4Yc+5zfWw==", + "requires": { + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-retry": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.3.tgz", + "integrity": "sha512-AFw+hjpbtVApzpNDhbjNG5NA3kyoMs7vx0gsgmlJF4s+yz1Zlepde7J58zpIRIsdjc+emhpAITxA88qLkPF26w==", + "requires": { + "@smithy/service-error-classification": "^3.0.3", + "@smithy/types": "^3.3.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-stream": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.1.3.tgz", + "integrity": "sha512-FIv/bRhIlAxC0U7xM1BCnF2aDRPq0UaelqBHkM2lsCp26mcBbgI0tCVTv+jGdsQLUmAMybua/bjDsSu8RQHbmw==", + "requires": { + "@smithy/fetch-http-handler": "^3.2.4", + "@smithy/node-http-handler": "^3.1.4", + "@smithy/types": "^3.3.0", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "requires": { + "tslib": "^2.6.2" + } + }, + "@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "requires": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + } + }, "@stylistic/eslint-plugin": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/@stylistic/eslint-plugin/-/eslint-plugin-2.3.0.tgz", @@ -10391,6 +12443,20 @@ "possible-typed-array-names": "^1.0.0" } }, + "aws4": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.13.1.tgz", + "integrity": "sha512-u5w79Rd7SU4JaIlA/zFqG+gOiuq25q5VLyZ8E+ijJeILuTxVzZgp2CaGw/UTw6pXYN9XMO9yiqj/nEHmhTG5CA==" + }, + "aws4-axios": { + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/aws4-axios/-/aws4-axios-3.3.7.tgz", + "integrity": "sha512-TVuUKe6SNv5vqrE6VLSR+eN8JJhijA28qKdMGKbY23TcksYl0KGrW9dADWRwLUiLTrKq/rTucjaoJzNruNf4RQ==", + "requires": { + "@aws-sdk/client-sts": "^3.4.1", + "aws4": "^1.12.0" + } + }, "axios": { "version": "1.7.2", "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", @@ -10502,6 +12568,11 @@ "resolved": "https://registry.npmjs.org/bit-buffer/-/bit-buffer-0.2.5.tgz", "integrity": "sha512-x1yGnmXvFg6e3DiyRztElbcn1bsCTFSoM/ncAzY62uE0JdTl5xlKJd0ooqLYoPbhdsnpehSIQrdIvclcZJYwiA==" }, + "bowser": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", + "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -11444,6 +13515,14 @@ } } }, + "eslint-plugin-jest": { + "version": "28.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jest/-/eslint-plugin-jest-28.8.0.tgz", + "integrity": "sha512-Tubj1hooFxCl52G4qQu0edzV/+EZzPUeN8p2NnW5uu4fbDs+Yo7+qDVDc4/oG3FbCqEBmu/OC3LSsyiU22oghw==", + "requires": { + "@typescript-eslint/utils": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, "eslint-plugin-license-header": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/eslint-plugin-license-header/-/eslint-plugin-license-header-0.6.1.tgz", @@ -11670,6 +13749,14 @@ "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, + "fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "requires": { + "strnum": "^1.0.5" + } + }, "fastq": { "version": "1.17.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", @@ -13755,6 +15842,11 @@ "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==" }, + "strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -14032,6 +16124,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" + }, "v8-compile-cache-lib": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", diff --git a/package.json b/package.json index d73f0868d..22d798e13 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", + "aws4-axios": "^3.3.7", "axios": "^1.7.1", "cbor": "^9.0.2", "commander": "^12.0.0", @@ -44,6 +45,7 @@ "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", "eslint-plugin-import": "^2.29.1", + "eslint-plugin-jest": "^28.8.0", "eslint-plugin-license-header": "^0.6.1", "eslint-plugin-n": "^16.6.2", "eslint-plugin-promise": "^6.1.1", diff --git a/tests/default/cat/plugins.yaml b/tests/default/cat/plugins.yaml index 383576a7b..a2fb2b42f 100644 --- a/tests/default/cat/plugins.yaml +++ b/tests/default/cat/plugins.yaml @@ -9,5 +9,3 @@ chapters: format: json response: status: 200 - payload: - - component: opensearch-alerting diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 2652cd101..f81656074 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -11,6 +11,8 @@ import { Option } from '@commander-js/extra-typings' import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios' import * as https from 'node:https' import { sleep } from './helpers' +import { Logger } from './Logger' +import { aws4Interceptor } from 'aws4-axios' const DEFAULT_URL = 'https://localhost:9200' const DEFAULT_USER = 'admin' @@ -30,23 +32,76 @@ export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password ', 'AWS access key ID') + .env('AWS_ACCESS_KEY_ID') + +export const AWS_SECRET_ACCESS_KEY_OPTION = new Option('--aws-secret-access-key ', 'AWS secret access key') + .env('AWS_SECRET_ACCESS_KEY') + +export const AWS_SESSION_TOKEN_OPTION = new Option('--aws-session-token ', 'AWS session token') + .env('AWS_SESSION_TOKEN') + +export const AWS_REGION_OPTION = new Option('--aws-region ', 'AWS region') + .env('AWS_REGION') + .default('us-east-1') + +export const AWS_SERVICE_OPTION = new Option('--aws-service ', 'AWS service ID') + .env('AWS_SERVICE') + .default('es') + +export interface BasicAuth { + username: string + password: string +} + +export interface AwsAuth { + aws_access_key_id: string + aws_access_secret_key: string + aws_access_session_token?: string + aws_region?: string + aws_service?: string +} + export interface OpenSearchHttpClientOptions { url?: string - username?: string - password?: string - insecure?: boolean, - responseType: ResponseType | undefined + insecure?: boolean + responseType?: ResponseType + logger?: Logger, + basic_auth?: BasicAuth + aws_auth?: AwsAuth } -export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize}`]: OpenSearchHttpClientOptions[K] } +export type OpenSearchHttpClientCliOptions = { + opensearchUrl?: string + opensearchUsername?: string + opensearchPassword?: string + opensearchInsecure?: boolean + awsAccessKeyId?: string + awsSecretAccessKey?: string + awsSessionToken?: string + awsRegion?: string + awsService?: string + responseType?: ResponseType + logger?: Logger +} export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptions): OpenSearchHttpClientOptions { return { url: opts.opensearchUrl, - username: opts.opensearchUsername, - password: opts.opensearchPassword, insecure: opts.opensearchInsecure, - responseType: opts.opensearchResponseType + basic_auth: opts.opensearchUsername !== undefined && opts.opensearchPassword !== undefined ? { + username: opts.opensearchUsername, + password: opts.opensearchPassword + } : undefined, + aws_auth: opts.awsAccessKeyId !== undefined && opts.awsSecretAccessKey !== undefined ? { + aws_access_key_id: opts?.awsAccessKeyId, + aws_access_secret_key: opts?.awsSecretAccessKey, + aws_access_session_token: opts?.awsSessionToken, + aws_region: opts?.awsRegion, + aws_service: opts?.awsService, + } : undefined, + responseType: opts.responseType, + logger: opts?.logger } } @@ -72,26 +127,52 @@ export interface OpenSearchInfo { export class OpenSearchHttpClient { private readonly _axios: AxiosInstance private readonly _opts?: OpenSearchHttpClientOptions + private readonly _logger: Logger constructor (opts?: OpenSearchHttpClientOptions) { this._opts = opts + this._logger = opts?.logger ?? new Logger() + + let auth = undefined + let sigv4_interceptor = undefined + + if (opts?.basic_auth !== undefined) { + this._logger.info(`Authenticating with ${opts.basic_auth.username} ...`) + auth = opts.basic_auth + } else if (opts?.aws_auth !== undefined) { + this._logger.info(`Authenticating using SigV4 with ${opts.aws_auth.aws_access_key_id} (${opts.aws_auth.aws_region}) ...`) + sigv4_interceptor = aws4Interceptor({ + options: { + region: opts.aws_auth.aws_region, + service: 'es' + }, + credentials: { + accessKeyId: opts.aws_auth.aws_access_key_id, + secretAccessKey: opts.aws_auth.aws_access_secret_key, + sessionToken: opts.aws_auth.aws_access_session_token, + } + }); + } else { + this._logger.warn(`No credentials provided, did you forget to set OPENSEARCH_PASSWORD or AWS_ACCESS_KEY_ID?`) + } + this._axios = axios.create({ baseURL: opts?.url ?? DEFAULT_URL, - auth: opts?.username !== undefined && opts.password !== undefined - ? { - username: opts.username, - password: opts.password - } - : undefined, + auth, httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), responseType: opts?.responseType, }) + + if (sigv4_interceptor !== undefined) { + this._axios.interceptors.request.use(sigv4_interceptor) + } } async wait_until_available (max_attempts: number = 20, wait_between_attempt_millis: number = 5000): Promise { let attempt = 0 while (true) { attempt += 1 + this._logger.info(`Connecting to ${this._opts?.url} ... (${attempt}/${max_attempts})`) try { const info = await this.get('/') if (this._opts?.responseType == 'arraybuffer') { @@ -101,9 +182,14 @@ export class OpenSearchHttpClient { } } catch (e) { if (axios.isAxiosError(e)) { + this._logger.warn(`Error connecting to ${this._opts?.url}: (${e.message})`) if (e.response?.status == 401 || e.code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') { - throw e + throw e.message + } else if (e.response?.status == 403 || e.code === 'ERR_BAD_REQUEST') { + throw e.message } + } else { + this._logger.warn(`Error connecting to ${this._opts?.url}: (${typeof (e)})`) } if (attempt >= max_attempts) { throw e diff --git a/tools/src/dump-cluster-spec/dump-cluster-spec.ts b/tools/src/dump-cluster-spec/dump-cluster-spec.ts index 1e65670e8..02dd77972 100644 --- a/tools/src/dump-cluster-spec/dump-cluster-spec.ts +++ b/tools/src/dump-cluster-spec/dump-cluster-spec.ts @@ -48,7 +48,7 @@ const command = new Command() const opts = command.opts() -main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ opensearchResponseType: undefined, ...opts }) }) +main({ output: opts.output, opensearch: get_opensearch_opts_from_cli({ responseType: undefined, ...opts }) }) .catch(e => { if (e instanceof Error) { console.error(`ERROR: ${e.stack}`) diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index a05800b37..134888d7f 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -11,6 +11,11 @@ import { Logger, LogLevel } from '../Logger' import TestRunner from './TestRunner' import { Command, Option } from '@commander-js/extra-typings' import { + AWS_ACCESS_KEY_ID_OPTION, + AWS_REGION_OPTION, + AWS_SECRET_ACCESS_KEY_OPTION, + AWS_SERVICE_OPTION, + AWS_SESSION_TOKEN_OPTION, get_opensearch_opts_from_cli, OPENSEARCH_INSECURE_OPTION, OPENSEARCH_PASSWORD_OPTION, @@ -46,6 +51,11 @@ const command = new Command() .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) .addOption(OPENSEARCH_INSECURE_OPTION) + .addOption(AWS_ACCESS_KEY_ID_OPTION) + .addOption(AWS_SECRET_ACCESS_KEY_OPTION) + .addOption(AWS_SESSION_TOKEN_OPTION) + .addOption(AWS_REGION_OPTION) + .addOption(AWS_SERVICE_OPTION) .addOption(new Option('--coverage ', 'path to write test coverage results to')) .allowExcessArguments(false) .parse() @@ -54,7 +64,7 @@ const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) -const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts })) +const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger) diff --git a/tools/tests/tester/OpenSearchHttpClient.test.ts b/tools/tests/tester/OpenSearchHttpClient.test.ts new file mode 100644 index 000000000..e2687e7fd --- /dev/null +++ b/tools/tests/tester/OpenSearchHttpClient.test.ts @@ -0,0 +1,66 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import axios from "axios"; +import { OpenSearchHttpClient } from "OpenSearchHttpClient" + +jest.mock('axios') + +describe('OpenSearchHttpClient', () => { + let mocked_axios: jest.Mocked = axios as jest.Mocked + + beforeEach(() => { + mocked_axios.create.mockReturnThis() + mocked_axios.interceptors.request.use = jest.fn().mockReturnValue({ headers: {} }) + }) + + afterEach(() => { + jest.clearAllMocks() + }) + + it('uses password authentication', () => { + new OpenSearchHttpClient({ + url: 'https://localhost:9200', + basic_auth: { + username: 'admin', + password: 'password' + } + }) + + expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ + auth: { + username: 'admin', + password: 'password' + }, + baseURL: 'https://localhost:9200' + }) + + expect(mocked_axios.interceptors.request.use).not.toHaveBeenCalled() + }) + + it('assigns a request interceptor with SigV4 authentication', () => { + new OpenSearchHttpClient({ + url: 'https://localhost:9200', + aws_auth: { + aws_access_key_id: 'key id', + aws_access_secret_key: 'secret key', + aws_access_session_token: 'session token', + aws_region: 'us-west-2', + aws_service: 'aoss' + } + }) + + expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ + auth: undefined, + baseURL: 'https://localhost:9200' + }) + + expect(mocked_axios.interceptors.request.use).toHaveBeenCalled() + }) +}) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index dd6758d2a..9e4cf2d46 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -41,8 +41,10 @@ export function construct_tester_components (spec_path: string): { const operation_locator = new OperationLocator(specification) const opensearch_http_client = new OpenSearchHttpClient({ insecure: true, - username: process.env.OPENSEARCH_USERNAME ?? 'admin', - password: process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!', + basic_auth: { + username: process.env.OPENSEARCH_USERNAME ?? 'admin', + password: process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + }, responseType: 'arraybuffer' }) const chapter_reader = new ChapterReader(opensearch_http_client, logger) From b50eb79ce746b4adf5988b99abdc50d875e82035 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 16:56:22 -0400 Subject: [PATCH 03/21] Run tests against stable OpenSearch 2.16. (#484) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 12 ++++++------ .github/workflows/test-tools-integ.yml | 2 +- spec/_info.yaml | 2 +- tools/tests/tester/fixtures/evals/passed.yaml | 2 +- .../tests/tester/fixtures/evals/skipped/semver.yaml | 2 +- tools/tests/tester/helpers.ts | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 3fa404483..b0014a322 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -27,17 +27,17 @@ jobs: admin_password: admin - version: 2.0.0 admin_password: admin - - version: 2.15.0 - - version: 2.15.0 + - version: 2.16.0 + - version: 2.16.0 tests: plugins/index_state_management - - version: 2.15.0 - tests: snapshot - version: 2.16.0 + tests: snapshot + - version: 2.17.0 hub: opensearchstaging - ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e' + ref: '@sha256:6398c27d7560626ed6b0ba28b3d6b20b7f00c6d94abf45ad3a820f8eeb3d61a3' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b' + ref: '@sha256:101681eea630393f8caf5987dd023a975a9656b63090a07bfdfe6ad2f73f0640' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/.github/workflows/test-tools-integ.yml b/.github/workflows/test-tools-integ.yml index 660a55e7e..5bb9ac5d8 100644 --- a/.github/workflows/test-tools-integ.yml +++ b/.github/workflows/test-tools-integ.yml @@ -24,7 +24,7 @@ jobs: test: runs-on: ubuntu-latest env: - OPENSEARCH_VERSION: 2.15.0 + OPENSEARCH_VERSION: 2.16.0 OPENSEARCH_PASSWORD: myStrongPassword123! OPENSEARCH_URL: https://localhost:9200 steps: diff --git a/spec/_info.yaml b/spec/_info.yaml index 2454ecb55..d8d63d088 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -2,4 +2,4 @@ $schema: ./json_schemas/_info.schema.yaml title: OpenSearch API Specification version: 1.0.0 -x-api-version: 2.15.0 +x-api-version: 2.16.0 diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index bb74087e8..12c1b2c32 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -175,7 +175,7 @@ chapters: - title: This GET /_cat/health should be skipped (> 2.999.0). overall: result: SKIPPED - message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. + message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/evals/skipped/semver.yaml b/tools/tests/tester/fixtures/evals/skipped/semver.yaml index b39460dcb..30709742d 100644 --- a/tools/tests/tester/fixtures/evals/skipped/semver.yaml +++ b/tools/tests/tester/fixtures/evals/skipped/semver.yaml @@ -3,5 +3,5 @@ full_path: tools/tests/tester/fixtures/stories/skipped/semver.yaml result: SKIPPED description: This story should be skipped because of version. -message: Skipped because version 2.15.0 does not satisfy >= 2.999.0. +message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 9e4cf2d46..ce11b6ad8 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.15.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) } From 6cdd8b7e86e2a2a08c07929f049d3ed17231b646 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 17:04:32 -0400 Subject: [PATCH 04/21] Fix: durations are not always integers. (#479) Signed-off-by: dblock --- CHANGELOG.md | 3 ++- spec/schemas/_common.yaml | 2 +- tools/tests/tester/fixtures/specs/excerpt.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5590437d..a0ecf4f75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `/_mapping` with `index` in query ([#385](https://github.com/opensearch-project/opensearch-api-specification/pull/385)) - Fixed duplicate `/_nodes/{node_id}` path ([#416](https://github.com/opensearch-project/opensearch-api-specification/pull/416)) - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430)) -- Fixed `_update_by_query` with a simple term ([451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) +- Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) +- Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) ### Security diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 1c20ad16b..5b645eed4 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -38,7 +38,7 @@ components: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value. - pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$ + pattern: ^([0-9\.]+)(?:d|h|m|s|ms|micros|nanos)$ type: string Metadata: type: object diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml index 64ad01d39..244e7e466 100644 --- a/tools/tests/tester/fixtures/specs/excerpt.yaml +++ b/tools/tests/tester/fixtures/specs/excerpt.yaml @@ -286,5 +286,5 @@ components: description: |- A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value. - pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$ + pattern: ^([0-9\.]+)(?:d|h|m|s|ms|micros|nanos)$ type: string \ No newline at end of file From ecf39215e18d9edeba1d048b2ba22a172844ae6c Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 8 Aug 2024 17:06:34 -0400 Subject: [PATCH 05/21] Fix: refresh index to get a consistent total. (#480) Signed-off-by: dblock --- tests/default/_core/reindex.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/default/_core/reindex.yaml b/tests/default/_core/reindex.yaml index 86d4fccb4..71567b7a9 100644 --- a/tests/default/_core/reindex.yaml +++ b/tests/default/_core/reindex.yaml @@ -39,6 +39,8 @@ chapters: - synopsis: Reindex a subset of documents (match). path: /_reindex method: POST + parameters: + refresh: true request: payload: source: @@ -82,7 +84,7 @@ chapters: response: status: 200 payload: - total: 1 + total: 2 - synopsis: Reindex only unique documents. path: /_reindex method: POST From d7e6971222d70f0219a01ba2ebb4941092d680de Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 11 Aug 2024 18:17:59 -0400 Subject: [PATCH 06/21] Double the RAM in the test container. (#486) Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index b0014a322..72a8d7601 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -47,7 +47,7 @@ jobs: OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }} OPENSEARCH_VERSION: ${{ matrix.entry.version }} OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }} - OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms2g -Xmx2g + OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g steps: - name: Checkout Repo From 56a5a224562d465f77c6e1712a266d1d11ba8a5c Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Sun, 11 Aug 2024 18:42:31 -0400 Subject: [PATCH 07/21] Document fielddata.yaml fail, ignore vim swp files (#473) * Document fielddata.yaml fail, ignore vim swp files * Ignore test files that start with . so vim swp files are ignored * fielddata.yaml test files if opensearch cluster was stopped with docker kill instead of docker stop, document the issue * upgrade remaining actions/checkout@v3 to v4 to stop warnings Signed-off-by: Andy Wick * fixed FAILED placment Signed-off-by: Andy Wick * switch to only processing .yaml files Signed-off-by: Andy Wick * Remove EXCLUDED_FILES instead only process .yaml Signed-off-by: Andy Wick --------- Signed-off-by: Andy Wick --- .github/workflows/changelog.yml | 2 +- .github/workflows/test-spec.yml | 2 +- TESTING_GUIDE.md | 4 ++++ tools/src/tester/TestRunner.ts | 11 ++++------- .../tester/fixtures/stories/ignore.wrong.extension | 1 + 5 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 tools/tests/tester/fixtures/stories/ignore.wrong.extension diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 23a1e0502..c6994060d 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -7,7 +7,7 @@ jobs: verify-changelog: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index 72a8d7601..d9ed81aea 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -95,7 +95,7 @@ jobs: if: github.event_name == 'pull_request' needs: test-opensearch-spec steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download Spec Coverage Data uses: actions/download-artifact@v4 diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 5c066549e..61f6c2828 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -6,6 +6,7 @@ - [Common Errors](#common-errors) - [401 Unauthorized](#401-unauthorized) - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api) + - [FAILED Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics) - [Writing Spec Tests](#writing-spec-tests) - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) @@ -85,6 +86,9 @@ curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_clu ' ``` +#### FAILED Cat with a json response (from security-analytics). +The cluster is not loading plugins correctly, maybe it was stopped using `docker kill` instead of `docker stop`. Recreating the cluster should fix the issue: `docker compose up --force-recreate -d`. + ## Writing Spec Tests The spec tests reside in the [tests/](tests) directory. Tests are organized in suites ([default](tests/default/), etc.), and subsequently in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/default/indices/index.yaml](tests/default/indices/index.yaml) (for `/{index}`), and [tests/default/indices/doc.yaml](tests/default/indices/doc.yaml) (for `/{index}/_doc`). diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index fdde53791..0c0f3c4ff 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -20,10 +20,6 @@ import { OpenSearchHttpClient } from 'OpenSearchHttpClient' import * as ansi from './Ansi' import _ from 'lodash' -const EXCLUDED_FILES = [ - 'docker-compose.yml' -] - export default class TestRunner { private readonly _http_client: OpenSearchHttpClient private readonly _story_validator: StoryValidator @@ -69,6 +65,9 @@ export default class TestRunner { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` if (fs.statSync(path).isFile()) { + if (!path.endsWith('.yaml')) { + return [] + } const story: Story = read_yaml(path) return [{ display_path: next_prefix === '' ? basename(path) : next_prefix, @@ -77,9 +76,7 @@ export default class TestRunner { }] } else { return _.compact(fs.readdirSync(path).flatMap(next_file => { - if (!EXCLUDED_FILES.includes(next_file)) { - return this.#collect_story_files(path, next_file, next_prefix) - } + return this.#collect_story_files(path, next_file, next_prefix) })) } } diff --git a/tools/tests/tester/fixtures/stories/ignore.wrong.extension b/tools/tests/tester/fixtures/stories/ignore.wrong.extension new file mode 100644 index 000000000..a649676ae --- /dev/null +++ b/tools/tests/tester/fixtures/stories/ignore.wrong.extension @@ -0,0 +1 @@ +Not a .yaml file, should be ignored From 00711ea55e7317840a6617bae872997ca4a14e63 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sun, 11 Aug 2024 19:20:56 -0400 Subject: [PATCH 08/21] Allow overwriting the Authorization header. (#488) Signed-off-by: dblock --- package-lock.json | 49 +++++++++++++++ package.json | 1 + tools/src/OpenSearchHttpClient.ts | 20 +++--- .../tests/tester/OpenSearchHttpClient.test.ts | 62 +++++++++++-------- 4 files changed, 98 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index e1156510e..00b73c87a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "ajv-formats": "^3.0.1", "aws4-axios": "^3.3.7", "axios": "^1.7.1", + "axios-mock-adapter": "^2.0.0", "cbor": "^9.0.2", "commander": "^12.0.0", "eslint": "^8.57.0", @@ -3935,6 +3936,18 @@ "proxy-from-env": "^1.1.0" } }, + "node_modules/axios-mock-adapter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.0.0.tgz", + "integrity": "sha512-D/K0J5Zm6KvaMTnsWrBQZWLzKN9GxUFZEa0mx2qeEHXDeTugCoplWehy8y36dj5vuSjhe1u/Dol8cZ8lzzmDew==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + }, + "peerDependencies": { + "axios": ">= 0.17.0" + } + }, "node_modules/babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -6423,6 +6436,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", @@ -12467,6 +12502,15 @@ "proxy-from-env": "^1.1.0" } }, + "axios-mock-adapter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-2.0.0.tgz", + "integrity": "sha512-D/K0J5Zm6KvaMTnsWrBQZWLzKN9GxUFZEa0mx2qeEHXDeTugCoplWehy8y36dj5vuSjhe1u/Dol8cZ8lzzmDew==", + "requires": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + } + }, "babel-jest": { "version": "29.7.0", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", @@ -14192,6 +14236,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, "is-builtin-module": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-3.2.1.tgz", diff --git a/package.json b/package.json index 22d798e13..5ffe41341 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "@types/titlecase": "^1.1.2", "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^6.21.0", + "axios-mock-adapter": "^2.0.0", "ajv": "^8.13.0", "ajv-errors": "^3.0.0", "ajv-formats": "^3.0.1", diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index f81656074..afbb7d5fa 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -133,18 +133,23 @@ export class OpenSearchHttpClient { this._opts = opts this._logger = opts?.logger ?? new Logger() - let auth = undefined - let sigv4_interceptor = undefined + let auth_middleware = undefined if (opts?.basic_auth !== undefined) { this._logger.info(`Authenticating with ${opts.basic_auth.username} ...`) - auth = opts.basic_auth + auth_middleware = ((request: any): any => { + if (request.headers.Authorization === undefined) { + const base64 = Buffer.from(`${opts.basic_auth?.username}:${opts.basic_auth?.password}`, 'utf8').toString('base64'); + request.headers.Authorization = `Basic ${base64}` + } + return request + }) } else if (opts?.aws_auth !== undefined) { this._logger.info(`Authenticating using SigV4 with ${opts.aws_auth.aws_access_key_id} (${opts.aws_auth.aws_region}) ...`) - sigv4_interceptor = aws4Interceptor({ + auth_middleware = aws4Interceptor({ options: { region: opts.aws_auth.aws_region, - service: 'es' + service: opts.aws_auth.aws_service }, credentials: { accessKeyId: opts.aws_auth.aws_access_key_id, @@ -158,13 +163,12 @@ export class OpenSearchHttpClient { this._axios = axios.create({ baseURL: opts?.url ?? DEFAULT_URL, - auth, httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), responseType: opts?.responseType, }) - if (sigv4_interceptor !== undefined) { - this._axios.interceptors.request.use(sigv4_interceptor) + if (auth_middleware !== undefined) { + this._axios.interceptors.request.use(auth_middleware) } } diff --git a/tools/tests/tester/OpenSearchHttpClient.test.ts b/tools/tests/tester/OpenSearchHttpClient.test.ts index e2687e7fd..85c62af05 100644 --- a/tools/tests/tester/OpenSearchHttpClient.test.ts +++ b/tools/tests/tester/OpenSearchHttpClient.test.ts @@ -9,58 +9,68 @@ import axios from "axios"; import { OpenSearchHttpClient } from "OpenSearchHttpClient" - -jest.mock('axios') +import AxiosMockAdapter from "axios-mock-adapter"; describe('OpenSearchHttpClient', () => { - let mocked_axios: jest.Mocked = axios as jest.Mocked + var mock = new AxiosMockAdapter(axios) - beforeEach(() => { - mocked_axios.create.mockReturnThis() - mocked_axios.interceptors.request.use = jest.fn().mockReturnValue({ headers: {} }) + afterEach(() => { + mock.reset() }) - afterEach(() => { - jest.clearAllMocks() + it('adds a Basic auth header', async () => { + let client = new OpenSearchHttpClient({ + url: 'https://localhost:9200', + basic_auth: { + username: 'u', + password: 'p' + } + }) + + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toMatch(/^Basic /) + return [200, { called: true }] + }) + + expect((await client.get('/')).data).toEqual({ called: true }) }) - it('uses password authentication', () => { - new OpenSearchHttpClient({ + it('allows to overwrite Authorization', async () => { + let client = new OpenSearchHttpClient({ url: 'https://localhost:9200', basic_auth: { - username: 'admin', - password: 'password' + username: 'u', + password: 'p' } }) - expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ - auth: { - username: 'admin', - password: 'password' - }, - baseURL: 'https://localhost:9200' + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toEqual('custom') + return [200, { called: true }] }) - expect(mocked_axios.interceptors.request.use).not.toHaveBeenCalled() + expect((await client.get('/', { headers: { Authorization: 'custom' } })).data).toEqual({ called: true }) }) - it('assigns a request interceptor with SigV4 authentication', () => { - new OpenSearchHttpClient({ + it('adds a Sigv4 header', async () => { + let client = new OpenSearchHttpClient({ url: 'https://localhost:9200', aws_auth: { aws_access_key_id: 'key id', aws_access_secret_key: 'secret key', aws_access_session_token: 'session token', - aws_region: 'us-west-2', + aws_region: 'us-west-42', aws_service: 'aoss' } }) - expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({ - auth: undefined, - baseURL: 'https://localhost:9200' + mock.onAny().reply((config) => { + expect(config.headers?.Authorization).toMatch( + /^AWS4-HMAC-SHA256 Credential=key id\/\d*\/us-west-42\/aoss\/aws4_request/ + ) + return [200, { called: true }] }) - expect(mocked_axios.interceptors.request.use).toHaveBeenCalled() + expect((await client.get('/')).data).toEqual({ called: true }) }) }) From 644a1a5b5f79ec0803ded1d408961a01e6deaccf Mon Sep 17 00:00:00 2001 From: Alen <78027095+aabeshov@users.noreply.github.com> Date: Mon, 12 Aug 2024 15:58:36 +0500 Subject: [PATCH 09/21] Observability namespace (#474) * Observability api and query api Signed-off-by: alen_abeshov * Fixing wrong linting Signed-off-by: alen_abeshov * added tests Signed-off-by: alen_abeshov * removed docker file Signed-off-by: alen_abeshov * Added tests Signed-off-by: alen_abeshov * Removed spaces, added exclude file Signed-off-by: alen_abeshov * Changed lycheeignore Signed-off-by: alen_abeshov * Fixing path Signed-off-by: alen_abeshov * Change URL Signed-off-by: alen_abeshov * add verbosity Signed-off-by: alen_abeshov * Checking version Signed-off-by: alen_abeshov * changin operation version Signed-off-by: alen_abeshov * Replacing http Signed-off-by: alen_abeshov * Changed desc Signed-off-by: alen_abeshov * Changed desc Signed-off-by: alen_abeshov * Fixing bug Signed-off-by: alen_abeshov --------- Signed-off-by: alen_abeshov Signed-off-by: Alen <78027095+aabeshov@users.noreply.github.com> --- .cspell | 3 +- .lycheeignore | 2 +- CHANGELOG.md | 1 + spec/namespaces/observability.yaml | 224 +++++++++++++++ spec/namespaces/query.yaml | 140 ++++++++++ spec/schemas/observability._common.yaml | 255 ++++++++++++++++++ spec/schemas/query._common.yaml | 131 +++++++++ .../default/observability/observability.yaml | 155 +++++++++++ tests/default/query/datasources.yaml | 95 +++++++ 9 files changed, 1004 insertions(+), 2 deletions(-) create mode 100644 spec/namespaces/observability.yaml create mode 100644 spec/namespaces/query.yaml create mode 100644 spec/schemas/observability._common.yaml create mode 100644 spec/schemas/query._common.yaml create mode 100644 tests/default/observability/observability.yaml create mode 100644 tests/default/query/datasources.yaml diff --git a/.cspell b/.cspell index 1542876a8..900b6c40a 100644 --- a/.cspell +++ b/.cspell @@ -185,4 +185,5 @@ urldecode vectory whoamiprotected wordnet -Yrtsd \ No newline at end of file +Yrtsd +localstats \ No newline at end of file diff --git a/.lycheeignore b/.lycheeignore index f437ebb39..c859dd4da 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -1 +1 @@ -https://localhost:9200* +https://localhost:* diff --git a/CHANGELOG.md b/CHANGELOG.md index a0ecf4f75..a941ea033 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462)) - Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465)) +- Added `observability` namespace API specifications ([#474](https://github.com/opensearch-project/opensearch-api-specification/pull/474)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) diff --git a/spec/namespaces/observability.yaml b/spec/namespaces/observability.yaml new file mode 100644 index 000000000..a8553dbe1 --- /dev/null +++ b/spec/namespaces/observability.yaml @@ -0,0 +1,224 @@ +openapi: 3.1.0 +info: + title: OpenSearch Observability Object API + description: API for retrieving and managing Observability Objects. + version: 1.0.0 +paths: + /_plugins/_observability/_local/stats: + get: + operationId: observability.get_localstats.0 + x-operation-group: observability.get_localstats + x-version-added: '1.1' + description: Retrieves Local Stats of all observability objects. + responses: + '200': + $ref: '#/components/responses/observability.get_localstats@200' + /_plugins/_observability/object: + get: + operationId: observability.list_objects.0 + x-operation-group: observability.list_objects + x-version-added: '1.1' + description: Retrieves list of all observability objects. + responses: + '200': + $ref: '#/components/responses/observability.list_objects@200' + post: + operationId: observability.create_object.0 + x-operation-group: observability.create_object + x-version-added: '1.1' + description: Creates a new observability object. + requestBody: + $ref: '#/components/requestBodies/observability.create_object' + responses: + '200': + $ref: '#/components/responses/observability.create_object@200' + delete: + operationId: observability.delete_objects.0 + x-operation-group: observability.delete_objects + x-version-added: '1.1' + description: Deletes specific observability objects specified by ID or a list of IDs. + parameters: + - $ref: '#/components/parameters/observability.delete_objects::query.objectId' + - $ref: '#/components/parameters/observability.delete_objects::query.objectIdList' + responses: + '200': + $ref: '#/components/responses/observability.delete_objects@200' + '404': + $ref: '#/components/responses/observability.delete_objects@404' + /_plugins/_observability/object/{object_id}: + get: + operationId: observability.get_object.0 + x-operation-group: observability.get_object + x-version-added: '1.1' + description: Retrieves specific observability object specified by ID. + parameters: + - $ref: '#/components/parameters/observability.get_object::path.object_id' + responses: + '200': + $ref: '#/components/responses/observability.get_object@200' + '404': + $ref: '#/components/responses/observability.get_object@404' + put: + operationId: observability.update_object.0 + x-operation-group: observability.update_object + x-version-added: '1.1' + description: Updates an existing observability object. + parameters: + - $ref: '#/components/parameters/observability.update_object::path.object_id' + requestBody: + $ref: '#/components/requestBodies/observability.update_object' + responses: + '200': + $ref: '#/components/responses/observability.update_object@200' + '404': + $ref: '#/components/responses/observability.update_object@404' + delete: + operationId: observability.delete_object.0 + x-operation-group: observability.delete_object + x-version-added: '1.1' + description: Deletes specific observability object specified by ID. + parameters: + - $ref: '#/components/parameters/observability.delete_object::path.object_id' + responses: + '200': + $ref: '#/components/responses/observability.delete_object@200' + '404': + $ref: '#/components/responses/observability.delete_object@404' +components: + requestBodies: + observability.create_object: + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObject' + observability.update_object: + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObject' + responses: + observability.list_objects@200: + description: Successful response of retrieving all Observability Objects. + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObjectList' + observability.create_object@200: + description: Created + content: + application/json: + schema: + type: object + properties: + objectId: + type: string + observability.get_object@200: + description: Successful response of retrieving specific Observability Object. + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/ObservabilityObjectList' + observability.get_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.update_object@200: + description: Updated + content: + application/json: + schema: + type: object + properties: + objectId: + type: string + observability.update_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.delete_object@200: + description: Deleted + content: + application/json: + schema: + type: object + properties: + deleteResponseList: + type: object + additionalProperties: + type: string + example: OK + observability.delete_object@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.delete_objects@200: + description: Deleted + content: + application/json: + schema: + type: object + properties: + deleteResponseList: + type: object + additionalProperties: + type: string + example: OK + observability.delete_objects@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/observability._common.yaml#/components/schemas/NotFoundResponse' + observability.get_localstats@200: + description: Retrieves + content: + application/json: + schema: + type: string + parameters: + observability.get_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.update_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.delete_object::path.object_id: + in: path + name: object_id + description: The ID of the Observability Object. + required: true + schema: + type: string + style: simple + observability.delete_objects::query.objectId: + in: query + name: objectId + description: The ID of a single Observability Object to delete. + required: false + schema: + type: string + style: form + observability.delete_objects::query.objectIdList: + in: query + name: objectIdList + description: A comma-separated list of Observability Object IDs to delete. + required: false + schema: + type: string + style: form diff --git a/spec/namespaces/query.yaml b/spec/namespaces/query.yaml new file mode 100644 index 000000000..b055f4666 --- /dev/null +++ b/spec/namespaces/query.yaml @@ -0,0 +1,140 @@ +openapi: 3.1.0 +info: + title: OpenSearch Query Datasources API + description: OpenSearch Query Datasources API. + version: 2.7.0 +paths: + /_plugins/_query/_datasources: + get: + operationId: query.datasources_list.0 + x-operation-group: query.datasources_list + x-version-added: '2.7' + description: Retrieves list of all datasources. + responses: + '200': + $ref: '#/components/responses/query.datasources_list@200' + post: + operationId: query.datasources_create.0 + x-operation-group: query.datasources_create + x-version-added: '2.7' + description: Creates a new query datasource. + requestBody: + $ref: '#/components/requestBodies/query.datasources_create' + responses: + '201': + $ref: '#/components/responses/query.datasources_create@201' + put: + operationId: query.datasources_update.0 + x-operation-group: query.datasources_update + x-version-added: '2.7' + description: Updates an existing query datasource. + requestBody: + $ref: '#/components/requestBodies/query.datasources_update' + responses: + '200': + $ref: '#/components/responses/query.datasources_update@200' + '404': + $ref: '#/components/responses/query.datasources_update@404' + /_plugins/_query/_datasources/{datasource_name}: + get: + operationId: query.datasource_retrieve.0 + x-operation-group: query.datasource_retrieve + x-version-added: '2.7' + description: Retrieves specific datasource specified by name. + parameters: + - $ref: '#/components/parameters/query.datasource_retrieve::path.datasource_name' + responses: + '200': + $ref: '#/components/responses/query.datasource_retrieve@200' + '404': + $ref: '#/components/responses/query.datasource_retrieve@404' + delete: + operationId: query.datasource_delete.0 + x-operation-group: query.datasource_delete + x-version-added: '2.7' + description: Deletes specific datasource specified by name. + parameters: + - $ref: '#/components/parameters/query.datasource_delete::path.datasource_name' + responses: + '204': + $ref: '#/components/responses/query.datasource_delete@204' + '404': + $ref: '#/components/responses/query.datasource_delete@404' + +components: + requestBodies: + query.datasources_create: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSource' + query.datasources_update: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSource' + responses: + query.datasources_list@200: + description: Successful response of retrieving all Data Sources. + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceList' + query.datasources_create@201: + description: Created + content: + application/json: + schema: + type: string + query.datasources_update@200: + description: Updated + content: + application/json: + schema: + type: string + query.datasources_update@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + query.datasource_retrieve@200: + description: Successful response of retrieving Data Source. + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceRetrieve' + query.datasource_retrieve@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + query.datasource_delete@204: + description: No Content + content: + application/json: + schema: + type: object + properties: { } + query.datasource_delete@404: + description: Not Found + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/DataSourceNotFound' + parameters: + query.datasource_delete::path.datasource_name: + name: datasource_name + in: path + description: The Name of the DataSource to delete. + schema: + type: string + required: true + query.datasource_retrieve::path.datasource_name: + name: datasource_name + in: path + description: The Name of the DataSource to retrieve. + schema: + type: string + required: true diff --git a/spec/schemas/observability._common.yaml b/spec/schemas/observability._common.yaml new file mode 100644 index 000000000..fc8018f55 --- /dev/null +++ b/spec/schemas/observability._common.yaml @@ -0,0 +1,255 @@ +openapi: 3.1.0 +info: + title: Schemas for OpenSearch Observability Object API + description: Schemas for OpenSearch Observability Object API + version: 1.0.0 +paths: {} +components: + schemas: + ObservabilityObjectList: + type: object + properties: + startIndex: + type: integer + totalHits: + type: integer + totalHitRelation: + type: string + observabilityObjectList: + type: array + items: + $ref: '#/components/schemas/ObservabilityObject' + required: + - observabilityObjectList + - startIndex + - totalHitRelation + - totalHits + + ObservabilityObject: + type: object + properties: + objectId: + type: string + lastUpdatedTimeMs: + type: integer + createdTimeMs: + type: integer + tenant: + type: string + operationalPanel: + $ref: '#/components/schemas/OperationalPanel' + savedVisualization: + $ref: '#/components/schemas/SavedVisualization' + savedQuery: + $ref: '#/components/schemas/SavedQuery' + required: + - objectId + - tenant + + OperationalPanel: + type: object + properties: + name: + type: string + visualizations: + type: array + items: + $ref: '#/components/schemas/Visualization' + timeRange: + $ref: '#/components/schemas/TimeRange' + queryFilter: + $ref: '#/components/schemas/QueryFilter' + applicationId: + type: string + required: + - applicationId + - name + - queryFilter + - timeRange + - visualizations + + Visualization: + type: object + properties: + id: + type: string + savedVisualizationId: + type: string + x: + type: integer + y: + type: integer + w: + type: integer + h: + type: integer + required: + - h + - id + - savedVisualizationId + - w + - x + - y + + TimeRange: + type: object + properties: + to: + type: string + from: + type: string + required: + - from + - to + + QueryFilter: + type: object + properties: + query: + type: string + language: + type: string + required: + - language + - query + + SavedVisualization: + type: object + properties: + name: + type: string + description: + type: string + query: + type: string + type: + type: string + selected_date_range: + $ref: '#/components/schemas/SelectedDateRange' + selected_timestamp: + $ref: '#/components/schemas/SelectedTimestamp' + selected_fields: + $ref: '#/components/schemas/SelectedFields' + required: + - description + - name + - query + - selected_date_range + - selected_fields + - selected_timestamp + - type + + SelectedDateRange: + type: object + properties: + start: + type: string + end: + type: string + text: + type: string + required: + - end + - start + - text + + SelectedTimestamp: + type: object + properties: + name: + type: string + type: + type: string + required: + - name + - type + + SelectedFields: + type: object + properties: + text: + type: string + tokens: + type: array + items: + $ref: '#/components/schemas/Token' + required: + - text + - tokens + + Token: + type: object + properties: + name: + type: string + type: + type: string + required: + - name + - type + + SavedQuery: + type: object + properties: + name: + type: string + description: + type: string + query: + type: string + selected_date_range: + $ref: '#/components/schemas/SelectedDateRange' + selected_timestamp: + $ref: '#/components/schemas/SelectedTimestamp' + selected_fields: + $ref: '#/components/schemas/SelectedFields' + required: + - description + - name + - query + - selected_date_range + - selected_fields + - selected_timestamp + + NotFoundResponse: + type: object + properties: + error: + $ref: '#/components/schemas/ErrorResponse' + status: + type: integer + example: 404 + required: + - error + - status + + ErrorResponse: + type: object + properties: + root_cause: + type: array + items: + $ref: '#/components/schemas/RootCause' + type: + type: string + example: status_exception + reason: + type: string + example: 'ObservabilityObject {objectId} not found' + required: + - reason + - root_cause + - type + + RootCause: + type: object + properties: + type: + type: string + example: status_exception + reason: + type: string + example: 'ObservabilityObject {objectId} not found' + required: + - reason + - type diff --git a/spec/schemas/query._common.yaml b/spec/schemas/query._common.yaml new file mode 100644 index 000000000..080c3afa3 --- /dev/null +++ b/spec/schemas/query._common.yaml @@ -0,0 +1,131 @@ +openapi: 3.1.0 +info: + title: Schemas for OpenSearch Query Datasources API + description: Schemas for OpenSearch Query Datasources API + version: 1.0.0 +paths: {} +components: + schemas: + DataSourceList: + type: array + items: + $ref: '#/components/schemas/DataSource' + + DataSource: + type: object + properties: + name: + type: string + description: + type: string + connector: + type: string + allowedRoles: + type: array + items: + type: string + properties: + type: object + additionalProperties: true + resultIndex: + type: string + status: + type: string + configuration: + $ref: '#/components/schemas/DataSourceConfiguration' + required: + - connector + - name + - properties + - resultIndex + - status + + DataSourceConfiguration: + type: object + properties: + endpoint: + type: string + credentials: + $ref: '#/components/schemas/Credentials' + required: + - credentials + - endpoint + + Credentials: + type: object + properties: + username: + type: string + password: + type: string + required: + - password + - username + + DataSourceNotFound: + type: object + properties: + error: + $ref: '#/components/schemas/ErrorResponse' + required: + - error + + ErrorResponse: + type: object + properties: + root_cause: + type: array + items: + $ref: '#/components/schemas/RootCause' + type: + type: string + example: status_exception + reason: + type: string + example: DataSource not found + required: + - reason + - root_cause + - type + + RootCause: + type: object + properties: + type: + type: string + example: status_exception + reason: + type: string + example: DataSource not found + required: + - reason + - type + + DataSourceRetrieve: + type: object + properties: + name: + type: string + description: + type: string + connector: + type: string + allowedRoles: + type: array + items: + type: string + properties: + type: object + additionalProperties: true + resultIndex: + type: string + status: + type: string + configuration: + $ref: '#/components/schemas/DataSourceConfiguration' + required: + - connector + - name + - properties + - resultIndex + - status diff --git a/tests/default/observability/observability.yaml b/tests/default/observability/observability.yaml new file mode 100644 index 000000000..f9e37461c --- /dev/null +++ b/tests/default/observability/observability.yaml @@ -0,0 +1,155 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test various operations of the OpenSearch Observability Object API. + +prologues: + - path: /_plugins/_observability/object/{object_id} + method: DELETE + parameters: + object_id: test_object + status: [200, 404] + - path: /_plugins/_observability/object + method: POST + request: + payload: + objectId: test_object + operationalPanel: + name: test_panel + visualizations: [] + timeRange: + from: now-1h + to: now + queryFilter: + query: '' + language: ppl + applicationId: test_app + savedVisualization: + name: viz1 + description: desc1 + query: '' + type: line + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + savedQuery: + name: query1 + description: desc1 + query: '' + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + status: [200] +chapters: + - synopsis: Retrieve specific Observability object after creation. + path: /_plugins/_observability/object/{object_id} + id: observatory_object + method: GET + parameters: + object_id: test_object + response: + status: 200 + payload: + startIndex: 0 + totalHits: 1 + totalHitRelation: eq + observabilityObjectList: [] + - synopsis: Update specific Observability object. + path: /_plugins/_observability/object/{object_id} + method: PUT + parameters: + object_id: test_object + request: + payload: + objectId: test_object + tenant: '' + operationalPanel: + name: updated_test_panel + visualizations: [] + timeRange: + from: now-1h + to: now + queryFilter: + query: '' + language: ppl + applicationId: test_app + savedVisualization: + name: '[Logs] Count total requests by tags' + description: '' + query: 'source = opensearch_dashboards_sample_data_logs | stats count() by tags' + type: bar + selected_date_range: + start: now/y + end: now + text: '' + selected_timestamp: + name: timestamp + type: timestamp + selected_fields: + text: '' + tokens: [] + savedQuery: + name: query1 + description: desc1 + query: '' + selected_date_range: + start: now-1d + end: now + text: Last 24 hours + selected_timestamp: + name: timestamp + type: time + selected_fields: + text: field1 + tokens: + - name: field1 + type: text + response: + status: 200 + payload: + objectId: test_object + - synopsis: Retrieve specific Observability object after update. + path: /_plugins/_observability/object/{object_id} + method: GET + parameters: + object_id: test_object + response: + status: 200 + payload: + startIndex: 0 + totalHits: 1 + totalHitRelation: eq + observabilityObjectList: [] + - synopsis: Retrieve list of Observability objects. + path: /_plugins/_observability/object + method: GET + response: + status: 200 + payload: + startIndex: 0 + totalHits: 0 + totalHitRelation: eq + observabilityObjectList: [] +epilogues: + - path: /_plugins/_observability/object/{object_id} + method: DELETE + parameters: + object_id: test_object + status: [200, 404] \ No newline at end of file diff --git a/tests/default/query/datasources.yaml b/tests/default/query/datasources.yaml new file mode 100644 index 000000000..35c9fea04 --- /dev/null +++ b/tests/default/query/datasources.yaml @@ -0,0 +1,95 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test various operations of the OpenSearch Query Datasources API. +version: '>=2.7' + +prologues: + - path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + status: [204, 404] +chapters: + - synopsis: Create a new query datasource. + path: /_plugins/_query/_datasources + method: POST + request: + payload: + name: test_datasource + description: '' + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9090' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + response: + status: 201 + payload: Created DataSource with name test_datasource + - synopsis: Retrieve the list of all query datasources. + path: /_plugins/_query/_datasources + method: GET + response: + status: 200 + payload: [] + - synopsis: Retrieve a specific query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: GET + parameters: + datasource_name: test_datasource + response: + status: 200 + payload: + name: test_datasource + description: '' + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9090' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + - synopsis: Update an existing query datasource. + path: /_plugins/_query/_datasources + method: PUT + request: + payload: + name: test_datasource + description: Updated description + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9091' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + response: + status: 200 + payload: Updated DataSource with name test_datasource + - synopsis: Retrieve the updated query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: GET + parameters: + datasource_name: test_datasource + response: + status: 200 + payload: + name: test_datasource + description: Updated description + connector: PROMETHEUS + allowedRoles: [] + properties: + prometheus.uri: 'https://localhost:9091' + resultIndex: query_execution_result_test_datasource + status: ACTIVE + - synopsis: Delete the query datasource by name. + path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + response: + status: 204 +epilogues: + - path: /_plugins/_query/_datasources/{datasource_name} + method: DELETE + parameters: + datasource_name: test_datasource + status: [204, 404] \ No newline at end of file From 597ed8d73c0b72743ed90fd9694ad9ba511d5be7 Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Mon, 12 Aug 2024 22:59:54 +1200 Subject: [PATCH 10/21] Improve indices.stats schemas (#491) * Improve indices.stats schemas Signed-off-by: Thomas Farr * Add changelog Signed-off-by: Thomas Farr * Remove unused schemas Signed-off-by: Thomas Farr --------- Signed-off-by: Thomas Farr --- CHANGELOG.md | 1 + spec/namespaces/indices.yaml | 8 +- spec/schemas/_common.yaml | 147 +++++++++--------------- spec/schemas/indices.stats.yaml | 195 +++++++++----------------------- spec/schemas/nodes._common.yaml | 2 +- 5 files changed, 118 insertions(+), 235 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a941ea033..a5821316f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430)) - Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) +- Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) ### Security diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index 6d36982f5..55f0c2999 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -2686,14 +2686,14 @@ components: schema: type: object properties: + _shards: + $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' + _all: + $ref: '../schemas/indices.stats.yaml#/components/schemas/AllIndicesStats' indices: type: object additionalProperties: $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' - _shards: - $ref: '../schemas/_common.yaml#/components/schemas/ShardStatistics' - _all: - $ref: '../schemas/indices.stats.yaml#/components/schemas/IndicesStats' required: - _all - _shards diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index 5b645eed4..f89bdb9bd 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -1250,13 +1250,6 @@ components: reserved_in_bytes: description: A prediction, in bytes, of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. type: number - total_data_set_size: - $ref: '#/components/schemas/ByteSize' - total_data_set_size_in_bytes: - description: |- - Total data set size, in bytes, of all shards assigned to selected nodes. - This includes the size of shards not stored fully on the nodes, such as the cache for partially mounted indices. - type: number required: - reserved_in_bytes - size_in_bytes @@ -1503,25 +1496,25 @@ components: GetStats: type: object properties: - current: + total: type: number - exists_time: + time: $ref: '#/components/schemas/Duration' - exists_time_in_millis: + time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' exists_total: type: number - missing_time: + exists_time: $ref: '#/components/schemas/Duration' - missing_time_in_millis: + exists_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' missing_total: type: number - time: + missing_time: $ref: '#/components/schemas/Duration' - time_in_millis: + missing_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - total: + current: type: number required: - current @@ -1534,40 +1527,39 @@ components: IndexingStats: type: object properties: + index_total: + type: number + index_time: + $ref: '#/components/schemas/Duration' + index_time_in_millis: + $ref: '#/components/schemas/DurationValueUnitMillis' index_current: type: number - delete_current: + index_failed: + type: number + delete_total: type: number delete_time: $ref: '#/components/schemas/Duration' delete_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - delete_total: + delete_current: type: number - doc_status: - $ref: '#/components/schemas/DocStatus' - is_throttled: - type: boolean noop_update_total: type: number + is_throttled: + type: boolean throttle_time: $ref: '#/components/schemas/Duration' throttle_time_in_millis: $ref: '#/components/schemas/DurationValueUnitMillis' - index_time: - $ref: '#/components/schemas/Duration' - index_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - index_total: - type: number - index_failed: - type: number + doc_status: + $ref: '#/components/schemas/DocStatus' types: + x-version-removed: '2.0' type: object additionalProperties: $ref: '#/components/schemas/IndexingStats' - write_load: - type: number required: - delete_current - delete_time_in_millis @@ -1684,6 +1676,21 @@ components: SearchStats: type: object properties: + open_contexts: + description: The number of open search contexts. + type: number + query_current: + description: The number of shard query operations that are currently running. + type: number + query_time: + description: The total amount of time for all shard query operations. + $ref: '#/components/schemas/Duration' + query_time_in_millis: + description: The total amount of time for all shard query operations, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + query_total: + description: The total number of shard query operations. + type: number concurrent_query_total: description: The total number of query operations that use concurrent segment search. type: number @@ -1710,39 +1717,6 @@ components: fetch_total: description: The total number of shard fetch operations. type: number - open_contexts: - description: The number of open search contexts. - type: number - point_in_time_total: - description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. - type: number - point_in_time_time: - $ref: '#/components/schemas/Duration' - point_in_time_time_in_millis: - description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - point_in_time_current: - description: The number of shard PIT contexts currently open. - type: number - query_current: - description: The number of shard query operations that are currently running. - type: number - query_time: - description: The total amount of time for all shard query operations. - $ref: '#/components/schemas/Duration' - query_time_in_millis: - description: The total amount of time for all shard query operations, in milliseconds. - $ref: '#/components/schemas/DurationValueUnitMillis' - query_total: - description: The total number of shard query operations. - type: number - request: - type: object - description: Statistics about coordinator search operations for the node. - additionalProperties: - $ref: '#/components/schemas/RequestStats' - search_idle_reactivate_count_total: - type: number scroll_current: description: The number of shard scroll operations that are currently running. type: number @@ -1755,6 +1729,17 @@ components: scroll_total: description: The total number of shard scroll operations. type: number + point_in_time_total: + description: The total number of shard Point in Time (PIT) contexts that have been created (completed and active) since the node last restarted. + type: number + point_in_time_time: + $ref: '#/components/schemas/Duration' + point_in_time_time_in_millis: + description: The amount of time that shard PIT contexts have been held open since the node last restarted, in milliseconds. + $ref: '#/components/schemas/DurationValueUnitMillis' + point_in_time_current: + description: The number of shard PIT contexts currently open. + type: number suggest_current: description: The number of shard suggest operations that are currently running. type: number @@ -1767,6 +1752,13 @@ components: suggest_total: description: The total number of shard suggest operations. type: number + search_idle_reactivate_count_total: + type: number + request: + type: object + description: Statistics about coordinator search operations for the node. + additionalProperties: + $ref: '#/components/schemas/RequestStats' groups: type: object additionalProperties: @@ -1833,33 +1825,6 @@ components: - current - total - total_time_in_millis - BulkStats: - type: object - properties: - total_operations: - type: number - total_time: - $ref: '#/components/schemas/Duration' - total_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - total_size: - $ref: '#/components/schemas/ByteSize' - total_size_in_bytes: - type: number - avg_time: - $ref: '#/components/schemas/Duration' - avg_time_in_millis: - $ref: '#/components/schemas/DurationValueUnitMillis' - avg_size: - $ref: '#/components/schemas/ByteSize' - avg_size_in_bytes: - type: number - required: - - avg_size_in_bytes - - avg_time_in_millis - - total_operations - - total_size_in_bytes - - total_time_in_millis GeoShapeRelation: type: string enum: diff --git a/spec/schemas/indices.stats.yaml b/spec/schemas/indices.stats.yaml index d3301fcd0..eb729afb7 100644 --- a/spec/schemas/indices.stats.yaml +++ b/spec/schemas/indices.stats.yaml @@ -24,130 +24,89 @@ components: required: - description - size_in_bytes + AllIndicesStats: + type: object + properties: + primaries: + $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' + required: + - primaries + - total IndicesStats: type: object properties: + uuid: + $ref: '_common.yaml#/components/schemas/Uuid' primaries: $ref: '#/components/schemas/IndexStats' + total: + $ref: '#/components/schemas/IndexStats' shards: type: object additionalProperties: type: array items: - $ref: '#/components/schemas/ShardStats' - total: - $ref: '#/components/schemas/IndexStats' - uuid: - $ref: '_common.yaml#/components/schemas/Uuid' - health: - $ref: '_common.yaml#/components/schemas/HealthStatus' - status: - $ref: '#/components/schemas/IndexMetadataState' - IndexStats: + $ref: '#/components/schemas/IndexShardStats' + required: + - primaries + - total + - uuid + IndexStatsBase: type: object properties: - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' docs: $ref: '_common.yaml#/components/schemas/DocStats' - fielddata: - $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' + store: + $ref: '_common.yaml#/components/schemas/StoreStats' indexing: $ref: '_common.yaml#/components/schemas/IndexingStats' - indices: - $ref: '#/components/schemas/IndicesStats' + get: + $ref: '_common.yaml#/components/schemas/GetStats' + search: + $ref: '_common.yaml#/components/schemas/SearchStats' merges: $ref: '_common.yaml#/components/schemas/MergesStats' - query_cache: - $ref: '_common.yaml#/components/schemas/QueryCacheStats' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' refresh: $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' - segments: - $ref: '_common.yaml#/components/schemas/SegmentsStats' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' - translog: - $ref: '_common.yaml#/components/schemas/TranslogStats' + flush: + $ref: '_common.yaml#/components/schemas/FlushStats' warmer: $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - ShardsTotalStats: - type: object - properties: - total_count: - type: number - required: - - total_count - ShardStats: - type: object - properties: - commit: - $ref: '#/components/schemas/ShardCommit' - completion: - $ref: '_common.yaml#/components/schemas/CompletionStats' - docs: - $ref: '_common.yaml#/components/schemas/DocStats' + query_cache: + $ref: '_common.yaml#/components/schemas/QueryCacheStats' fielddata: $ref: '_common.yaml#/components/schemas/FielddataStats' - flush: - $ref: '_common.yaml#/components/schemas/FlushStats' - get: - $ref: '_common.yaml#/components/schemas/GetStats' - indexing: - $ref: '_common.yaml#/components/schemas/IndexingStats' - mappings: - $ref: '#/components/schemas/MappingStats' - merges: - $ref: '_common.yaml#/components/schemas/MergesStats' - shard_path: - $ref: '#/components/schemas/ShardPath' - query_cache: - $ref: '#/components/schemas/ShardQueryCache' - recovery: - $ref: '_common.yaml#/components/schemas/RecoveryStats' - refresh: - $ref: '_common.yaml#/components/schemas/RefreshStats' - request_cache: - $ref: '_common.yaml#/components/schemas/RequestCacheStats' - retention_leases: - $ref: '#/components/schemas/ShardRetentionLeases' - routing: - $ref: '#/components/schemas/ShardRouting' - search: - $ref: '_common.yaml#/components/schemas/SearchStats' + completion: + $ref: '_common.yaml#/components/schemas/CompletionStats' segments: $ref: '_common.yaml#/components/schemas/SegmentsStats' - seq_no: - $ref: '#/components/schemas/ShardSequenceNumber' - store: - $ref: '_common.yaml#/components/schemas/StoreStats' translog: $ref: '_common.yaml#/components/schemas/TranslogStats' - warmer: - $ref: '_common.yaml#/components/schemas/WarmerStats' - bulk: - $ref: '_common.yaml#/components/schemas/BulkStats' - shards: - type: object - additionalProperties: - type: object - shard_stats: - $ref: '#/components/schemas/ShardsTotalStats' - indices: - $ref: '#/components/schemas/IndicesStats' + request_cache: + $ref: '_common.yaml#/components/schemas/RequestCacheStats' + recovery: + $ref: '_common.yaml#/components/schemas/RecoveryStats' + IndexStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + IndexShardStats: + allOf: + - $ref: '#/components/schemas/IndexStatsBase' + - type: object + properties: + routing: + $ref: '#/components/schemas/ShardRouting' + commit: + $ref: '#/components/schemas/ShardCommit' + seq_no: + $ref: '#/components/schemas/ShardSequenceNumber' + retention_leases: + $ref: '#/components/schemas/ShardRetentionLeases' + shard_path: + $ref: '#/components/schemas/ShardPath' ShardCommit: type: object properties: @@ -166,18 +125,6 @@ components: - id - num_docs - user_data - MappingStats: - type: object - properties: - total_count: - type: number - total_estimated_overhead: - $ref: '_common.yaml#/components/schemas/ByteSize' - total_estimated_overhead_in_bytes: - type: number - required: - - total_count - - total_estimated_overhead_in_bytes ShardPath: type: object properties: @@ -191,31 +138,6 @@ components: - data_path - is_custom_data_path - state_path - ShardQueryCache: - type: object - properties: - cache_count: - type: number - cache_size: - type: number - evictions: - type: number - hit_count: - type: number - memory_size_in_bytes: - type: number - miss_count: - type: number - total_count: - type: number - required: - - cache_count - - cache_size - - evictions - - hit_count - - memory_size_in_bytes - - miss_count - - total_count ShardRetentionLeases: type: object properties: @@ -281,9 +203,4 @@ components: required: - global_checkpoint - local_checkpoint - - max_seq_no - IndexMetadataState: - type: string - enum: - - close - - open + - max_seq_no \ No newline at end of file diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index 59378e88a..ff0efdc21 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -250,7 +250,7 @@ components: indexing_pressure: $ref: '#/components/schemas/IndexingPressure' indices: - $ref: 'indices.stats.yaml#/components/schemas/ShardStats' + $ref: 'indices.stats.yaml#/components/schemas/IndexShardStats' shard_indexing_pressure: $ref: '#/components/schemas/ShardIndexingPressureStats' search_backpressure: From cb320b5482551c4f28afa26ff0d1653332699722 Mon Sep 17 00:00:00 2001 From: Andy Wick Date: Mon, 12 Aug 2024 11:26:49 -0400 Subject: [PATCH 11/21] Request sort in rest_total_hits_as_int tests (#493) Signed-off-by: Andy Wick --- .../_core/search/rest_total_hits_as_int.yaml | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 3d01d28da..8e7a04351 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -11,6 +11,7 @@ prologues: director: Bennett Miller title: Moneyball year: 2011 + order: 1 status: [201] - path: /movies/_doc method: POST @@ -21,6 +22,7 @@ prologues: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 status: [201] epilogues: - path: /movies @@ -37,26 +39,26 @@ chapters: payload: query: match_all: {} + sort: order response: status: 200 payload: timed_out: false hits: total: 2 - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 - synopsis: Search with rest_total_hits_as_int=false. path: /{index}/_search parameters: @@ -67,6 +69,7 @@ chapters: payload: query: match_all: {} + sort: order response: status: 200 payload: @@ -75,20 +78,19 @@ chapters: total: value: 2 relation: eq - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 - synopsis: Search with rest_total_hits_as_int=false track_total_hits=1. path: /{index}/_search parameters: @@ -100,6 +102,7 @@ chapters: track_total_hits: 1 query: match_all: {} + sort: order response: status: 200 payload: @@ -108,18 +111,17 @@ chapters: total: value: 1 relation: gte - max_score: 1 hits: - _index: movies - _score: 1 _source: director: Bennett Miller title: Moneyball year: 2011 + order: 1 - _index: movies - _score: 1 _source: director: Nicolas Winding Refn title: Drive year: 2011 + order: 2 From 382883848467c7f2edef81c074446c73e88bb80f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 13:03:20 -0400 Subject: [PATCH 12/21] Catch response deserialization errors. (#478) * Catch unexpected errors. Signed-off-by: dblock * Eat the error instead of catching it. Signed-off-by: dblock * Distinguish errors and errors. Signed-off-by: dblock --------- Signed-off-by: dblock --- tools/src/tester/ChapterEvaluator.ts | 10 +++-- tools/src/tester/ChapterReader.ts | 18 ++++---- .../fixtures/evals/error/chapter_error.yaml | 1 - .../fixtures/evals/error/prologue_error.yaml | 1 - .../fixtures/stories/failed/not_found.yaml | 1 - .../tests/tester/integ/StoryEvaluator.test.ts | 44 +++++++++++++++++++ 6 files changed, 59 insertions(+), 16 deletions(-) diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 0be147f4b..ae73f90a3 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -122,15 +122,17 @@ export default class ChapterEvaluator { #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation { const expected_status = chapter.response?.status ?? 200 - if (response.status === expected_status) return { result: Result.PASSED } + if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED } - const result: Evaluation = { + let result: Evaluation = { result: Result.ERROR, message: _.join(_.compact([ - `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`, + expected_status == response.status ? + `Received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.` : + `Expected status ${expected_status}, but received ${response.status ?? 'none'}: ${response.content_type ?? 'unknown'}.`, response.message ]), ' ') - }; + } if (response.error !== undefined) { result.error = response.error as Error diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index ae26e9920..26dd02753 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -56,16 +56,16 @@ export default class ChapterReader { }).catch(e => { if (e.response == null) { this.logger.info(`<= ERROR: ${e}`) - throw e + response.message = e.message + response.error = e + } else { + response.status = e.response.status + response.content_type = e.response.headers['content-type']?.split(';')[0] + const payload = this.#deserialize_payload(e.response.data, response.content_type) + if (payload !== undefined) response.payload = payload.error + response.message = payload.error?.reason ?? e.response.statusText + this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) } - response.status = e.response.status - response.content_type = e.response.headers['content-type']?.split(';')[0] - const payload = this.#deserialize_payload(e.response.data, response.content_type) - if (payload !== undefined) response.payload = payload.error - response.message = payload.error?.reason ?? e.response.statusText - response.error = e - - this.logger.info(`<= ${response.status} (${response.content_type}) | ${response.payload !== undefined ? to_json(response.payload) : response.message}`) }) return response as ActualResponse } diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index 450d9c5a3..fab1f0cc4 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -33,7 +33,6 @@ chapters: result: ERROR message: 'Expected status 200, but received 404: application/json. no such index [undefined]' - error: Request failed with status code 404 payload_body: result: SKIPPED payload_schema: diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml index 5ff63bb87..d91beb82f 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml @@ -16,7 +16,6 @@ prologues: overall: result: ERROR message: no such index [does_not_exists] - error: Request failed with status code 404 chapters: - title: This chapter be skipped. diff --git a/tools/tests/tester/fixtures/stories/failed/not_found.yaml b/tools/tests/tester/fixtures/stories/failed/not_found.yaml index 0b61b03a9..2bedb9c68 100644 --- a/tools/tests/tester/fixtures/stories/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/stories/failed/not_found.yaml @@ -29,4 +29,3 @@ chapters: index: movies response: status: 404 - diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index 7abc9dbfd..ac5fc60ef 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -7,6 +7,7 @@ * compatible open source license. */ +import { Result } from 'tester/types/eval.types' import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from '../helpers' const { story_evaluator, opensearch_http_client } = construct_tester_components('tools/tests/tester/fixtures/specs/excerpt.yaml') @@ -16,6 +17,10 @@ beforeAll(async () => { expect(info.version).toBeDefined() }) +afterEach(() => { + jest.resetAllMocks() +}) + test('passed', async () => { const actual = await load_actual_evaluation(story_evaluator, 'passed') const expected = load_expected_evaluation('passed') @@ -57,3 +62,42 @@ test('skipped/semver', async () => { const expected = load_expected_evaluation('skipped/semver') expect(actual).toEqual(expected) }) + +test('with an unexpected error deserializing data', async () => { + opensearch_http_client.request = jest.fn().mockRejectedValue(new Error('This was unexpected.')) + const actual = await load_actual_evaluation(story_evaluator, 'passed') + expect(actual.result).toEqual(Result.ERROR) + expect(actual.chapters && actual.chapters[0]).toEqual({ + title: "This PUT /{index} chapter should pass.", + path: 'PUT /{index}', + overall: { + result: Result.ERROR + }, + request: { + parameters: { + index: { + result: Result.PASSED + }, + }, + request: { + result: Result.PASSED + } + }, + response: { + output_values: { + result: Result.SKIPPED + }, + payload_body: { + result: Result.SKIPPED + }, + payload_schema: { + result: Result.SKIPPED + }, + status: { + error: 'This was unexpected.', + message: 'Expected status 200, but received none: unknown. This was unexpected.', + result: Result.ERROR + } + } + }) +}) \ No newline at end of file From 9eb37325f5ec46dd2c204d0c933ddfc413056e4a Mon Sep 17 00:00:00 2001 From: Darshit Chanpura Date: Mon, 12 Aug 2024 14:10:40 -0400 Subject: [PATCH 13/21] Adds test for APIs related to security plugin and updates spec to add new APIs. (#439) * Updates the missing item types for security API related schemas Signed-off-by: Darshit Chanpura * Adds tests for some APIs Signed-off-by: Darshit Chanpura * Fixes schema path and lint error in a file Signed-off-by: Darshit Chanpura * Adds version check for account API test Signed-off-by: Darshit Chanpura * Adds a CHANGELOG entry Signed-off-by: Darshit Chanpura * Fixes tests to expect status as integer Signed-off-by: Darshit Chanpura * Adds base tests for all APIs Signed-off-by: Darshit Chanpura * Fixes linter errors and complete payloads and request bodies Signed-off-by: Darshit Chanpura * Adds API spec for the new certificates API and adds tests Signed-off-by: Darshit Chanpura * Adds missing new lines at the end of files and removes any extra lines added by linter Signed-off-by: Darshit Chanpura * Fixes lint errors Signed-off-by: Darshit Chanpura * Fixes test spec lint Signed-off-by: Darshit Chanpura * Fixes without api prefix tests Signed-off-by: Darshit Chanpura * Fixes accounts tests Signed-off-by: Darshit Chanpura * Fixes action-groups tests Signed-off-by: Darshit Chanpura * Fixes account, allowlist and audit tests Signed-off-by: Darshit Chanpura * Fixes certificates tests Signed-off-by: Darshit Chanpura * Fixes internal users test Signed-off-by: Darshit Chanpura * Fixes nodesdn test Signed-off-by: Darshit Chanpura * Fixes roles test Signed-off-by: Darshit Chanpura * Fixes rolesmapping test Signed-off-by: Darshit Chanpura * Fixes securityconfig test Signed-off-by: Darshit Chanpura * Fixes ssl_certs tests Signed-off-by: Darshit Chanpura * Fixes tenancyconfig tests Signed-off-by: Darshit Chanpura * Fixes tenants tests Signed-off-by: Darshit Chanpura * Fixes tokens and upgrade tests Signed-off-by: Darshit Chanpura * Fixes user tests Signed-off-by: Darshit Chanpura * Fixes validate tests and run linter Signed-off-by: Darshit Chanpura * Fixes spec Signed-off-by: Darshit Chanpura * Adds missing new lines to EOFs Signed-off-by: Darshit Chanpura * Address changes in CHANGELOG Signed-off-by: Darshit Chanpura * Updates parameter descriptions Signed-off-by: Darshit Chanpura * Re-organize folder structure Signed-off-by: Darshit Chanpura * Moved security tests inside plugins folder Signed-off-by: Darshit Chanpura * Updates test spec workflow to run security tests Signed-off-by: Darshit Chanpura * Adds security test specific docker compose file Signed-off-by: Darshit Chanpura * Cleans commented code and updates test-spec to run for 2.16 only Signed-off-by: Darshit Chanpura * Refactor s boolean to be true boolean Signed-off-by: Darshit Chanpura * Move security tests to default folder and updates spec file Signed-off-by: Darshit Chanpura * Fixes 1.3 tests Signed-off-by: Darshit Chanpura * Fixes 2.0 test failures Signed-off-by: Darshit Chanpura * Fixes TLS cipher versiion payload issue Signed-off-by: Darshit Chanpura * Re-verifies auth token fix Signed-off-by: Darshit Chanpura * Checks for newer images and pulls them before running tests Signed-off-by: Darshit Chanpura * Updates the docker compose command Signed-off-by: Darshit Chanpura * Removes sha ref for staging branches Signed-off-by: Darshit Chanpura * Updates shas Signed-off-by: Darshit Chanpura * Renames everything to match the API path Signed-off-by: Darshit Chanpura --------- Signed-off-by: Darshit Chanpura --- .cspell | 1 + .github/workflows/test-spec.yml | 4 +- CHANGELOG.md | 3 + spec/namespaces/security.yaml | 417 ++++++++++++++---- spec/schemas/security._common.yaml | 214 ++++++--- .../default/security/api/_upgrade_check.yaml | 11 + .../security/api/_upgrade_perform.yaml | 14 + tests/default/security/api/account.yaml | 21 + tests/default/security/api/actiongroups.yaml | 59 +++ tests/default/security/api/allowlist.yaml | 31 ++ tests/default/security/api/audit.yaml | 20 + tests/default/security/api/audit/config.yaml | 39 ++ tests/default/security/api/authtoken.yaml | 10 + tests/default/security/api/cache.yaml | 25 ++ tests/default/security/api/certificates.yaml | 35 ++ .../security/api/generateonbehalfoftoken.yaml | 18 + tests/default/security/api/internalusers.yaml | 60 +++ .../api/internalusers.yml/authtoken.yml | 35 ++ tests/default/security/api/migrate.yaml | 11 + tests/default/security/api/nodesdn.yaml | 61 +++ .../default/security/api/permissionsinfo.yaml | 10 + tests/default/security/api/roles.yaml | 75 ++++ tests/default/security/api/rolesmapping.yaml | 77 ++++ .../default/security/api/securityconfig.yaml | 22 + .../security/api/securityconfig/config.yaml | 43 ++ tests/default/security/api/ssl/certs.yml | 12 + .../security/api/ssl/http/reloadcerts.yaml | 12 + .../api/ssl/transport/reloadcerts.yaml | 12 + .../default/security/api/tenancy/config.yaml | 22 + tests/default/security/api/tenants.yaml | 56 +++ tests/default/security/api/user.yaml | 37 ++ .../default/security/api/user/authtoken.yaml | 35 ++ tests/default/security/api/validate.yaml | 13 + tests/default/security/authinfo.yaml | 34 ++ tests/default/security/dashboardsinfo.yaml | 15 + tests/default/security/health.yaml | 27 ++ tests/default/security/sslinfo.yaml | 25 ++ tests/default/security/tenantinfo.yaml | 17 + tests/default/security/whoami.yaml | 24 + tests/default/security/whoamiprotected.yaml | 15 + 40 files changed, 1503 insertions(+), 169 deletions(-) create mode 100644 tests/default/security/api/_upgrade_check.yaml create mode 100644 tests/default/security/api/_upgrade_perform.yaml create mode 100644 tests/default/security/api/account.yaml create mode 100644 tests/default/security/api/actiongroups.yaml create mode 100644 tests/default/security/api/allowlist.yaml create mode 100644 tests/default/security/api/audit.yaml create mode 100644 tests/default/security/api/audit/config.yaml create mode 100644 tests/default/security/api/authtoken.yaml create mode 100644 tests/default/security/api/cache.yaml create mode 100644 tests/default/security/api/certificates.yaml create mode 100644 tests/default/security/api/generateonbehalfoftoken.yaml create mode 100644 tests/default/security/api/internalusers.yaml create mode 100644 tests/default/security/api/internalusers.yml/authtoken.yml create mode 100644 tests/default/security/api/migrate.yaml create mode 100644 tests/default/security/api/nodesdn.yaml create mode 100644 tests/default/security/api/permissionsinfo.yaml create mode 100644 tests/default/security/api/roles.yaml create mode 100644 tests/default/security/api/rolesmapping.yaml create mode 100644 tests/default/security/api/securityconfig.yaml create mode 100644 tests/default/security/api/securityconfig/config.yaml create mode 100644 tests/default/security/api/ssl/certs.yml create mode 100644 tests/default/security/api/ssl/http/reloadcerts.yaml create mode 100644 tests/default/security/api/ssl/transport/reloadcerts.yaml create mode 100644 tests/default/security/api/tenancy/config.yaml create mode 100644 tests/default/security/api/tenants.yaml create mode 100644 tests/default/security/api/user.yaml create mode 100644 tests/default/security/api/user/authtoken.yaml create mode 100644 tests/default/security/api/validate.yaml create mode 100644 tests/default/security/authinfo.yaml create mode 100644 tests/default/security/dashboardsinfo.yaml create mode 100644 tests/default/security/health.yaml create mode 100644 tests/default/security/sslinfo.yaml create mode 100644 tests/default/security/tenantinfo.yaml create mode 100644 tests/default/security/whoami.yaml create mode 100644 tests/default/security/whoamiprotected.yaml diff --git a/.cspell b/.cspell index 900b6c40a..1fcd3199c 100644 --- a/.cspell +++ b/.cspell @@ -165,6 +165,7 @@ subqueries subschemas subword syserr +tcnative tdigest tenantinfo termvectors diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index d9ed81aea..13a218e29 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -34,10 +34,10 @@ jobs: tests: snapshot - version: 2.17.0 hub: opensearchstaging - ref: '@sha256:6398c27d7560626ed6b0ba28b3d6b20b7f00c6d94abf45ad3a820f8eeb3d61a3' + ref: '@sha256:ed4274522a50228f41b50f1a7ea86e6b52fa6737072fc151b2624d22aff80d56' - version: 3.0.0 hub: opensearchstaging - ref: '@sha256:101681eea630393f8caf5987dd023a975a9656b63090a07bfdfe6ad2f73f0640' + ref: '@sha256:cab6f71b284485c44306f8f4849ad520283c2a32ece617109b38183ba29cc401' name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }}) runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index a5821316f..25a085e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -68,6 +68,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `observability` namespace API specifications ([#474](https://github.com/opensearch-project/opensearch-api-specification/pull/474)) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) +- Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) ### Changed @@ -111,6 +113,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451)) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) +- Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) ### Security diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index f41647f59..72da8905e 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -60,8 +60,6 @@ paths: x-operation-group: security.post_dashboards_info x-version-added: '1.0' description: Updates the current security-dashboards plugin configuration. - requestBody: - $ref: '#/components/requestBodies/security.post_dashboards_info' responses: '200': $ref: '#/components/responses/security.post_dashboards_info@200' @@ -101,6 +99,8 @@ paths: responses: '200': $ref: '#/components/responses/security.tenant_info@200' + '403': + $ref: '#/components/responses/security.tenant_info@403' '500': $ref: '#/components/responses/security.tenant_info@500' post: @@ -111,13 +111,15 @@ paths: responses: '200': $ref: '#/components/responses/security.tenant_info@200' + '403': + $ref: '#/components/responses/security.tenant_info@403' '500': $ref: '#/components/responses/security.tenant_info@500' /_plugins/_security/whoami: get: operationId: security.who_am_i.0 x-operation-group: security.who_am_i - x-version-added: '1.0' + x-version-added: '2.0' description: Gets the user identity related information for currently logged in user. responses: '200': @@ -127,7 +129,7 @@ paths: post: operationId: security.who_am_i.1 x-operation-group: security.who_am_i - x-version-added: '1.0' + x-version-added: '2.0' description: Gets the user identity related information for currently logged in user. responses: '200': @@ -145,7 +147,7 @@ paths: $ref: '#/components/responses/security.who_am_i_protected@200' '500': $ref: '#/components/responses/security.who_am_i_protected@500' - /_plugins/_security/_upgrade_check: + /_plugins/_security/api/_upgrade_check: get: operationId: security.config_upgrade_check.0 x-operation-group: security.config_upgrade_check @@ -156,7 +158,7 @@ paths: responses: '200': $ref: '#/components/responses/security.config_upgrade_check@200' - /_plugins/_security/_upgrade_perform: + /_plugins/_security/api/_upgrade_perform: post: operationId: security.config_upgrade_perform.0 x-operation-group: security.config_upgrade_perform @@ -169,6 +171,8 @@ paths: responses: '200': $ref: '#/components/responses/security.config_upgrade_perform@200' + '400': + $ref: '#/components/responses/security.config_upgrade_perform@400' /_plugins/_security/api/account: get: operationId: security.get_account_details.0 @@ -192,6 +196,8 @@ paths: responses: '200': $ref: '#/components/responses/security.change_password@200' + '403': + $ref: '#/components/responses/security.change_password@403' /_plugins/_security/api/actiongroups: get: operationId: security.get_action_groups.0 @@ -242,6 +248,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_action_group@200' + '201': + $ref: '#/components/responses/security.create_action_group@201' patch: operationId: security.patch_action_group.0 x-operation-group: security.patch_action_group @@ -272,17 +280,19 @@ paths: get: operationId: security.get_allowlist.0 x-operation-group: security.get_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Retrieves the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api responses: '200': $ref: '#/components/responses/security.get_allowlist@200' + '403': + $ref: '#/components/responses/security.get_allowlist@403' put: operationId: security.create_allowlist.0 x-operation-group: security.create_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Creates or replaces the allowlisted APIs. Accessible via Super Admin certificate or REST API permission. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -291,10 +301,12 @@ paths: responses: '200': $ref: '#/components/responses/security.create_allowlist@200' + '403': + $ref: '#/components/responses/security.create_allowlist@403' patch: operationId: security.patch_allowlist.0 x-operation-group: security.patch_allowlist - x-version-added: '1.0' + x-version-added: '2.1' description: Updates the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -303,6 +315,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_allowlist@200' + '403': + $ref: '#/components/responses/security.patch_allowlist@403' /_plugins/_security/api/audit: get: operationId: security.get_audit_configuration.0 @@ -386,6 +400,39 @@ paths: responses: '200': $ref: '#/components/responses/security.flush_cache@200' + /_plugins/_security/api/certificates: + get: + operationId: security.get_all_certificates.0 + x-operation-group: security.get_all_certificates + x-version-added: '2.15' + description: Retrieves the cluster security certificates. + parameters: + - $ref: '#/components/parameters/security.get_all_certificates::query.cert_type' + - $ref: '#/components/parameters/security.get_all_certificates::query.timeout' + responses: + '200': + $ref: '#/components/responses/security.get_all_certificates@200' + '403': + $ref: '#/components/responses/security.get_all_certificates@403' + '500': + $ref: '#/components/responses/security.get_all_certificates@500' + /_plugins/_security/api/certificates/{node_id}: + get: + operationId: security.get_node_certificates.0 + x-operation-group: security.get_node_certificates + x-version-added: '2.15' + description: Retrieves the given node's security certificates. + parameters: + - $ref: '#/components/parameters/security.get_node_certificates::path.node_id' + - $ref: '#/components/parameters/security.get_node_certificates::query.cert_type' + - $ref: '#/components/parameters/security.get_node_certificates::query.timeout' + responses: + '200': + $ref: '#/components/responses/security.get_node_certificates@200' + '403': + $ref: '#/components/responses/security.get_node_certificates@403' + '500': + $ref: '#/components/responses/security.get_node_certificates@500' /_plugins/_security/api/generateonbehalfoftoken: post: operationId: security.generate_obo_token.0 @@ -481,7 +528,7 @@ paths: post: operationId: security.generate_user_token.0 x-operation-group: security.generate_user_token - x-version-added: '1.0' + x-version-added: '2.7' description: Generates authorization token for the given user. parameters: - $ref: '#/components/parameters/security.generate_user_token::path.username' @@ -514,6 +561,8 @@ paths: responses: '200': $ref: '#/components/responses/security.get_distinguished_names@200' + '400': + $ref: '#/components/responses/security.get_distinguished_names@400' '403': $ref: '#/components/responses/security.get_distinguished_names@403' patch: @@ -528,6 +577,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_distinguished_names@200' + '400': + $ref: '#/components/responses/security.patch_distinguished_names@400' '403': $ref: '#/components/responses/security.patch_distinguished_names@403' /_plugins/_security/api/nodesdn/{cluster_name}: @@ -544,6 +595,8 @@ paths: responses: '200': $ref: '#/components/responses/security.get_distinguished_name@200' + '400': + $ref: '#/components/responses/security.get_distinguished_name@400' '403': $ref: '#/components/responses/security.get_distinguished_name@403' put: @@ -560,6 +613,8 @@ paths: responses: '200': $ref: '#/components/responses/security.update_distinguished_name@200' + '400': + $ref: '#/components/responses/security.update_distinguished_name@400' '403': $ref: '#/components/responses/security.update_distinguished_name@403' patch: @@ -574,6 +629,8 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_distinguished_name@200' + '400': + $ref: '#/components/responses/security.patch_distinguished_name@400' '403': $ref: '#/components/responses/security.patch_distinguished_name@403' delete: @@ -588,6 +645,8 @@ paths: responses: '200': $ref: '#/components/responses/security.delete_distinguished_name@200' + '400': + $ref: '#/components/responses/security.delete_distinguished_name@400' '403': $ref: '#/components/responses/security.delete_distinguished_name@403' /_plugins/_security/api/permissionsinfo: @@ -653,6 +712,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_role@200' + '201': + $ref: '#/components/responses/security.create_role@201' patch: operationId: security.patch_role.0 x-operation-group: security.patch_role @@ -733,6 +794,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_role_mapping@200' + '201': + $ref: '#/components/responses/security.create_role_mapping@201' patch: operationId: security.patch_role_mapping.0 x-operation-group: security.patch_role_mapping @@ -765,7 +828,7 @@ paths: get: operationId: security.get_configuration.0 x-operation-group: security.get_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: Returns the current Security plugin configuration in JSON format. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-configuration @@ -775,7 +838,7 @@ paths: patch: operationId: security.patch_configuration.0 x-operation-group: security.patch_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: A PATCH call is used to update the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-configuration @@ -784,11 +847,13 @@ paths: responses: '200': $ref: '#/components/responses/security.patch_configuration@200' + '403': + $ref: '#/components/responses/security.patch_configuration@403' /_plugins/_security/api/securityconfig/config: put: operationId: security.update_configuration.0 x-operation-group: security.update_configuration - x-version-added: '1.0' + x-version-added: '2.10' description: Adds or updates the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#update-configuration @@ -797,11 +862,13 @@ paths: responses: '200': $ref: '#/components/responses/security.update_configuration@200' + '403': + $ref: '#/components/responses/security.update_configuration@403' /_plugins/_security/api/ssl/certs: get: operationId: security.get_certificates.0 x-operation-group: security.get_certificates - x-version-added: '1.0' + x-version-added: '2.0' description: Retrieves the cluster security certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-certificates @@ -810,11 +877,13 @@ paths: $ref: '#/components/responses/security.get_certificates@200' '400': $ref: '#/components/responses/security.get_certificates@400' + '403': + $ref: '#/components/responses/security.get_certificates@403' /_plugins/_security/api/ssl/http/reloadcerts: put: operationId: security.reload_http_certificates.0 x-operation-group: security.reload_http_certificates - x-version-added: '1.0' + x-version-added: '2.8' description: Reload HTTP layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-http-certificates @@ -823,11 +892,13 @@ paths: $ref: '#/components/responses/security.reload_http_certificates@200' '400': $ref: '#/components/responses/security.reload_http_certificates@400' + '403': + $ref: '#/components/responses/security.reload_http_certificates@403' /_plugins/_security/api/ssl/transport/reloadcerts: put: operationId: security.reload_transport_certificates.0 x-operation-group: security.reload_transport_certificates - x-version-added: '1.0' + x-version-added: '2.8' description: Reload Transport layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-transport-certificates @@ -836,6 +907,8 @@ paths: $ref: '#/components/responses/security.reload_transport_certificates@200' '400': $ref: '#/components/responses/security.reload_transport_certificates@400' + '403': + $ref: '#/components/responses/security.reload_transport_certificates@403' /_plugins/_security/api/tenancy/config: get: operationId: security.get_tenancy_config.0 @@ -917,6 +990,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_tenant@200' + '201': + $ref: '#/components/responses/security.create_tenant@201' '400': $ref: '#/components/responses/security.create_tenant@400' patch: @@ -981,6 +1056,8 @@ paths: responses: '200': $ref: '#/components/responses/security.create_user_legacy@200' + '201': + $ref: '#/components/responses/security.create_user_legacy@201' delete: operationId: security.delete_user_legacy.0 x-operation-group: security.delete_user_legacy @@ -995,15 +1072,13 @@ paths: post: operationId: security.generate_user_token_legacy.0 x-operation-group: security.generate_user_token_legacy - x-version-added: '1.0' - description: Generates authorization token for the given user. Legacy API. + x-version-added: '2.7' + description: Generates authorization token for the given user. Legacy API. Not Implemented. parameters: - $ref: '#/components/parameters/security.generate_user_token_legacy::path.username' responses: - '200': - $ref: '#/components/responses/security.generate_user_token_legacy@200' - '400': - $ref: '#/components/responses/security.generate_user_token_legacy@400' + '501': + $ref: '#/components/responses/security.generate_user_token_legacy@501' /_plugins/_security/api/validate: get: operationId: security.validate.0 @@ -1064,9 +1139,7 @@ components: content: application/json: schema: - type: array - items: - $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/MultiTenancyConfig' required: true security.create_user: content: @@ -1203,12 +1276,6 @@ components: items: $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' required: true - security.post_dashboards_info: - content: - application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/DashboardsInfo' - required: false security.update_audit_configuration: content: application/json: @@ -1225,7 +1292,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' + $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' responses: security.authinfo@200: content: @@ -1245,13 +1312,16 @@ components: security.cache@501: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/MethodNotImplemented' + schema: null security.change_password@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.change_password@403: + content: + application/json: + schema: null security.config_upgrade_check@200: content: application/json: @@ -1262,36 +1332,63 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/UpgradePerform' + security.config_upgrade_perform@400: + content: + application/json: + schema: null security.create_action_group@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_action_group@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_allowlist@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.create_allowlist@403: + content: + application/json: + schema: null security.create_role@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_role@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_role_mapping@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_role_mapping@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.create_tenant@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' - security.create_tenant@400: + security.create_tenant@201: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + $ref: '../schemas/security._common.yaml#/components/schemas/Created' + security.create_tenant@400: + content: + application/json: + schema: null security.create_update_tenancy_config@200: content: application/json: @@ -1300,8 +1397,7 @@ components: security.create_update_tenancy_config@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.create_user@200: content: application/json: @@ -1312,6 +1408,11 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.create_user_legacy@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Created' security.delete_action_group@200: content: application/json: @@ -1322,11 +1423,14 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.delete_distinguished_name@400: + content: + application/json: + schema: null security.delete_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.delete_role@200: content: application/json: @@ -1345,8 +1449,7 @@ components: security.delete_tenant@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.delete_user@200: content: application/json: @@ -1369,9 +1472,10 @@ components: $ref: '../schemas/security._common.yaml#/components/schemas/GenerateOBOToken' security.generate_obo_token@400: content: + text/plain: + type: string application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.generate_user_token@200: content: application/json: @@ -1380,18 +1484,11 @@ components: security.generate_user_token@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' - security.generate_user_token_legacy@200: - content: - application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Ok' - security.generate_user_token_legacy@400: + schema: null + security.generate_user_token_legacy@501: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_account_details@200: content: application/json: @@ -1412,6 +1509,10 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.get_allowlist@403: + content: + application/json: + schema: null security.get_audit_configuration@200: content: application/json: @@ -1422,16 +1523,47 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificates' + security.get_all_certificates@200: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificatesNew' + security.get_all_certificates@403: + content: + application/json: + schema: null + security.get_all_certificates@500: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' + security.get_node_certificates@200: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/GetCertificatesNew' + security.get_node_certificates@403: + content: + application/json: + schema: null security.get_certificates@400: + content: + application/json: + schema: null + security.get_certificates@403: + content: + application/json: + schema: null + security.get_node_certificates@500: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + $ref: '../schemas/security._common.yaml#/components/schemas/InternalServerError' security.get_configuration@200: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/DynamicConfig' + $ref: '../schemas/security._common.yaml#/components/schemas/SecurityConfig' security.get_dashboards_info@200: content: application/json: @@ -1448,21 +1580,28 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' + security.get_distinguished_name@400: + content: + application/json: + schema: null security.get_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.get_distinguished_names@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNamesMap' + security.get_distinguished_names@400: + description: Show nodesDn setting for given cluster. + content: + application/json: + schema: null security.get_distinguished_names@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.get_permissions_info@200: content: application/json: @@ -1516,8 +1655,7 @@ components: security.get_tenancy_config@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_tenants@200: content: application/json: @@ -1526,8 +1664,7 @@ components: security.get_tenants@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.get_user@200: content: application/json: @@ -1561,8 +1698,7 @@ components: security.migrate@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_action_group@200: content: application/json: @@ -1578,6 +1714,10 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/AllowListConfig' + security.patch_allowlist@403: + content: + application/json: + schema: null security.patch_audit_configuration@200: content: application/json: @@ -1588,26 +1728,36 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_configuration@403: + content: + application/json: + schema: null security.patch_distinguished_name@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_distinguished_name@400: + content: + application/json: + schema: null security.patch_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.patch_distinguished_names@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.patch_distinguished_names@400: + content: + application/json: + schema: null security.patch_distinguished_names@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.patch_role@200: content: application/json: @@ -1616,8 +1766,7 @@ components: security.patch_role@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_role_mapping@200: content: application/json: @@ -1626,8 +1775,7 @@ components: security.patch_role_mapping@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_role_mappings@200: content: application/json: @@ -1636,8 +1784,7 @@ components: security.patch_role_mappings@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_roles@200: content: application/json: @@ -1646,8 +1793,7 @@ components: security.patch_roles@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_tenant@200: content: application/json: @@ -1656,8 +1802,7 @@ components: security.patch_tenant@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_tenants@200: content: application/json: @@ -1666,8 +1811,7 @@ components: security.patch_tenants@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.patch_user@200: content: application/json: @@ -1696,8 +1840,11 @@ components: security.reload_http_certificates@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null + security.reload_http_certificates@403: + content: + application/json: + schema: null security.reload_transport_certificates@200: content: application/json: @@ -1706,13 +1853,22 @@ components: security.reload_transport_certificates@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null + security.reload_transport_certificates@403: + content: + application/json: + schema: null security.tenant_info@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/TenantInfo' + security.tenant_info@403: + content: + text/plain: + type: string + application/json: + schema: null security.tenant_info@500: content: application/json: @@ -1728,16 +1884,23 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.update_configuration@403: + content: + application/json: + schema: null security.update_distinguished_name@200: content: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.update_distinguished_name@400: + content: + application/json: + schema: null security.update_distinguished_name@403: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/Unauthorized' + schema: null security.validate@200: content: application/json: @@ -1746,8 +1909,7 @@ components: security.validate@400: content: application/json: - schema: - $ref: '../schemas/security._common.yaml#/components/schemas/BadRequest' + schema: null security.who_am_i@200: content: application/json: @@ -1789,117 +1951,165 @@ components: description: The name of the action group to create or replace. schema: type: string - description: The name of the action group to create or replace. required: true security.create_role::path.role: name: role in: path + description: The name of the role to be created. schema: type: string required: true security.create_role_mapping::path.role: name: role in: path + description: The name of the role to create a role mapping for. schema: type: string required: true security.create_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to be created. schema: type: string required: true security.create_user::path.username: name: username in: path + description: The name of the user to be created. schema: type: string required: true security.create_user_legacy::path.username: name: username in: path + description: The name of the user to be created. schema: type: string required: true security.delete_action_group::path.action_group: name: action_group in: path - description: Action group to delete. + description: The name of the action group to delete. schema: type: string - description: Action group to delete. required: true security.delete_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to delete from list of distinguished names. schema: type: string required: true security.delete_role::path.role: name: role in: path + description: The name of the role to delete. schema: type: string required: true security.delete_role_mapping::path.role: name: role in: path + description: The name of the role whose mapping needs to delete. schema: type: string required: true security.delete_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to delete. schema: type: string required: true security.delete_user::path.username: name: username in: path + description: The name of the user to delete. schema: type: string required: true security.delete_user_legacy::path.username: name: username in: path + description: The name of the user to delete. schema: type: string required: true security.generate_user_token::path.username: name: username in: path + description: The name of the user for whom an auth token is to be vended. schema: type: string required: true security.generate_user_token_legacy::path.username: name: username in: path + description: The name of the user for whom an auth token is to be vended. schema: type: string required: true security.get_action_group::path.action_group: name: action_group in: path - description: Action group to retrieve. + description: The name of the action group to retrieve. schema: type: string - description: Action group to retrieve. required: true + security.get_node_certificates::path.node_id: + name: node_id + in: path + description: The full-id of the node to retrieve certificates. + schema: + type: string + required: true + security.get_all_certificates::query.cert_type: + name: cert_type + in: query + description: The type of certificates (HTTP, TRANSPORT, ALL) to retrieve from all nodes. + schema: + type: string + required: false + security.get_node_certificates::query.cert_type: + name: cert_type + in: query + description: The type of certificates (HTTP, TRANSPORT, ALL) to retrieve for a node. + schema: + type: string + required: false + security.get_all_certificates::query.timeout: + name: timeout + in: query + description: The maximum duration, in seconds, to be spent to retrieve certificates from all nodes. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + required: false + security.get_node_certificates::query.timeout: + name: timeout + in: query + description: The maximum duration, in seconds, to be spent to retrieve a node's certificates. + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Duration' + required: false security.get_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to retrieve nodes DN setting for. schema: type: string required: true security.get_distinguished_name::query.show_all: name: show_all in: query + description: A boolean flag to include/exclude static nodes DN from final result. schema: type: boolean required: false security.get_distinguished_names::query.show_all: name: show_all in: query + description: A boolean flag to include/exclude static nodes DN from final result. schema: type: boolean required: false @@ -1918,80 +2128,91 @@ components: security.get_sslinfo::query.show_dn: name: show_dn in: query - description: The domain names from all certificates. + description: A boolean flag to indicate whether all domain names should be returned. schema: - type: string - description: A boolean flag to indicate whether all domain names should be returned. + type: [boolean, string] required: false security.get_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to retrieve. schema: type: string required: true security.get_user::path.username: name: username in: path + description: The name of the user to retrieve. schema: type: string required: true security.get_user_legacy::path.username: name: username in: path + description: The name of the user to retrieve. schema: type: string required: true security.health::query.mode: name: mode in: query + description: A flag to indicate whether service should consider security-plugin's status before returning health response. `strict` mode indicates service should check security plugin status. schema: type: string required: false security.patch_action_group::path.action_group: name: action_group in: path + description: The name of the action group to update. schema: type: string required: true security.patch_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to update nodesDn value. schema: type: string required: true security.patch_role::path.role: name: role in: path + description: The name of the role to update. schema: type: string required: true security.patch_role_mapping::path.role: name: role in: path + description: The name of the role to update role-mapping for. schema: type: string required: true security.patch_tenant::path.tenant: name: tenant in: path + description: The name of the tenant to update. schema: type: string required: true security.patch_user::path.username: name: username in: path + description: The name of the user to update. schema: type: string required: true security.update_distinguished_name::path.cluster_name: name: cluster_name in: path + description: The cluster-name to create/update nodesDn value for. schema: type: string required: true security.validate::query.accept_invalid: name: accept_invalid in: query + description: A boolean flag to indicate whether invalid v6 configuration should be allowed. schema: type: boolean required: false diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index 2fcce2ec2..36c976cb0 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -18,7 +18,7 @@ components: is_internal_user: type: boolean user_requested_tenant: - type: string + type: ['null', string] backend_roles: type: array items: @@ -58,12 +58,6 @@ components: $ref: '#/components/schemas/ActionGroup' AllowListConfig: - type: object - properties: - config: - $ref: '#/components/schemas/AllowConfig' - - AllowConfig: type: object properties: enabled: @@ -103,6 +97,14 @@ components: type: array items: type: string + ignore_headers: + type: array + items: + type: string + ignore_url_params: + type: array + items: + type: string disabled_rest_categories: type: array items: @@ -134,31 +136,37 @@ components: type: string description: User's name. user_requested_tenant: - type: string + type: ['null', string] description: Name of the tenant the user wants to switch to. remote_address: - type: string + type: ['null', string] description: The IP address of remote user. backend_roles: type: array description: Backend roles associated with the user. + items: + type: string custom_attribute_names: type: array description: Name of the attributes associated with the user. + items: + type: string roles: type: array description: Roles associated with the user. + items: + type: string tenants: type: object description: Tenants the user has access to with read-write or read-only access indicator. principal: - type: string + type: ['null', string] description: User principal. peer_certificates: - type: number + type: [number, string] description: Number of peer certificates. sso_logout_url: - type: string + type: ['null', string] description: Logout url. size_of_user: type: string @@ -170,16 +178,18 @@ components: type: string description: Size of backend roles in bytes. - BadRequest: + CertificateCountPerNode: type: object properties: - status: - type: string - enum: - - 400 - message: - type: string - description: Message returned as part of BAD_REQUEST response. + total: + type: number + description: Total number of nodes. + successful: + type: number + description: Number of nodes for which certificates could be fetched. + failed: + type: number + description: Number of nodes for which certificates could not be fetched. CertificatesDetail: type: object @@ -194,7 +204,34 @@ components: type: string not_after: type: string + + CertificatesPerNode: + type: object + properties: + name: + type: string + description: Name of the node. + certificates: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificateTypes' + CertificateTypes: + type: object + properties: + http: + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesDetail' + transport: + type: array + items: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesDetail' + GetCertificates: type: object properties: @@ -206,6 +243,22 @@ components: type: array items: $ref: '#/components/schemas/CertificatesDetail' + + GetCertificatesNew: + type: object + properties: + _nodes: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificateCountPerNode' + cluster_name: + type: string + description: Name of this cluster. + nodes: + type: object + additionalProperties: + $ref: '#/components/schemas/CertificatesPerNode' + ChangePasswordRequestContent: type: object @@ -255,6 +308,8 @@ components: config: type: array description: List of configs to be upgraded. + items: + type: string CreateTenantParams: type: object @@ -292,6 +347,8 @@ components: sign_in_options: type: array description: List of available sign-in options available. + items: + type: string password_validation_error_message: type: string description: Error message when password validation fails. @@ -321,27 +378,36 @@ components: DynamicOptions: type: object properties: - filteredAliasMode: + filtered_alias_mode: type: string - disableRestAuth: + disable_rest_auth: type: boolean - disableIntertransportAuth: + disable_intertransport_auth: type: boolean - respectRequestIndicesOptions: + respect_request_indices_options: type: boolean - kibana: {} - http: {} - authc: {} - authz: {} - authFailureListeners: {} - doNotFailOnForbidden: + opensearch-dashboards: + type: object + kibana: + type: object + http: + type: object + authc: + type: object + authz: + type: object + auth_failure_listeners: + type: object + do_not_fail_on_forbidden: type: boolean - multiRolespanEnabled: + multi_rolespan_enabled: type: boolean - hostsResolverMode: + hosts_resolver_mode: type: string - doNotFailOnForbiddenEmpty: + do_not_fail_on_forbidden_empty: type: boolean + on_behalf_of: + type: object GenerateOBOToken: type: object @@ -360,7 +426,7 @@ components: type: object properties: message: - type: string + type: ['null', string] mode: type: string status: @@ -395,17 +461,6 @@ components: type: string description: Error message during request execution. - MethodNotImplemented: - type: object - properties: - status: - type: string - enum: - - 501 - message: - type: string - description: Message returned as part of NOT_IMPLEMENTED response. - MultiTenancyConfig: type: object properties: @@ -419,7 +474,6 @@ components: type: array items: type: string - description: Value in seconds. OBOToken: type: object @@ -440,12 +494,19 @@ components: type: object properties: status: - type: string - enum: - - 200 + type: [number, string] message: type: string description: Message returned as part of OK response. + + Created: + type: object + properties: + status: + type: [number, string] + message: + type: string + description: Message returned as part of CREATED response. PatchOperation: type: object @@ -534,22 +595,32 @@ components: type: object additionalProperties: $ref: '#/components/schemas/Role' + + SecurityConfig: + type: object + properties: + config: + $ref: '#/components/schemas/DynamicConfig' SSLInfo: type: object properties: principal: - type: string + type: ['null', string] description: User principal. peer_certificates: - type: number + type: [number, string] description: Number of certificates. peer_certificates_list: - type: array + type: [array,'null'] description: List of domain names from peer certificates. + items: + type: string local_certificates_list: type: array description: List of domain names from local certificates. + items: + type: string ssl_protocol: type: string description: Protocol for this ssl setup. @@ -560,13 +631,13 @@ components: type: boolean description: A boolean to indicate if OpenSSL is available. ssl_openssl_version: - type: string + type: [number, string] description: Version of openssl. ssl_openssl_version_string: - type: string + type: ['null', string] description: Full version string for openssl version. ssl_openssl_non_available_cause: - type: string + type: ['null', string] description: Reason for openssl unavailability. ssl_openssl_supports_key_manager_factory: type: boolean @@ -583,6 +654,20 @@ components: ssl_provider_transport_client: type: string description: Returns transport client's name. + required: + - peer_certificates + - principal + - ssl_cipher + - ssl_openssl_available + - ssl_openssl_non_available_cause + - ssl_openssl_supports_hostname_validation + - ssl_openssl_supports_key_manager_factory + - ssl_openssl_version + - ssl_openssl_version_string + - ssl_protocol + - ssl_provider_http + - ssl_provider_transport_client + - ssl_provider_transport_server Tenant: type: object @@ -613,17 +698,6 @@ components: additionalProperties: $ref: '#/components/schemas/Tenant' - Unauthorized: - type: object - properties: - status: - type: string - enum: - - 403 - message: - type: string - description: Message returned as part of FORBIDDEN response. - UpgradeCheck: type: object properties: @@ -645,6 +719,8 @@ components: User: type: object properties: + password: + type: string hash: type: string reserved: @@ -690,11 +766,11 @@ components: type: object properties: dn: - type: string + type: ['null', string] is_admin: - type: string + type: boolean is_node_certificate_request: - type: string + type: boolean TenantInfo: type: object diff --git a/tests/default/security/api/_upgrade_check.yaml b/tests/default/security/api/_upgrade_check.yaml new file mode 100644 index 000000000..60d553753 --- /dev/null +++ b/tests/default/security/api/_upgrade_check.yaml @@ -0,0 +1,11 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test _upgrade_check endpoint. +version: '> 2.13' + +chapters: + - synopsis: Check whether an upgrade can be performed. + path: /_plugins/_security/api/_upgrade_check + method: GET + response: + status: 200 diff --git a/tests/default/security/api/_upgrade_perform.yaml b/tests/default/security/api/_upgrade_perform.yaml new file mode 100644 index 000000000..643f7896f --- /dev/null +++ b/tests/default/security/api/_upgrade_perform.yaml @@ -0,0 +1,14 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test _upgrade_perform endpoint. +version: '> 2.13' + +chapters: + - synopsis: Perform the upgrade. + path: /_plugins/_security/api/_upgrade_perform + method: POST + request: + payload: + config: [roles] + response: + status: 400 # Unable to upgrade, no differences found in 'roles' config. diff --git a/tests/default/security/api/account.yaml b/tests/default/security/api/account.yaml new file mode 100644 index 000000000..2fc10499f --- /dev/null +++ b/tests/default/security/api/account.yaml @@ -0,0 +1,21 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test account endpoint. + +chapters: + - synopsis: Get account details. + path: /_plugins/_security/api/account + method: GET + response: + status: 200 + # TODO: following test can be changed to interact with test user, to be created in prologues, once https://github.com/opensearch-project/opensearch-api-specification/issues/438 is implemented + # NOTE: this test should be updated to change test user's password created in epilogue. Until then this will be 403 since admin is a reserved user. + - synopsis: Change password. + path: /_plugins/_security/api/account + method: PUT + request: + payload: + current_password: myStrongPassword123! + password: myWeakPassword123! + response: + status: 403 diff --git a/tests/default/security/api/actiongroups.yaml b/tests/default/security/api/actiongroups.yaml new file mode 100644 index 000000000..0321d9022 --- /dev/null +++ b/tests/default/security/api/actiongroups.yaml @@ -0,0 +1,59 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test actiongroups endpoints. + +chapters: + - synopsis: Create action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: PUT + parameters: + action_group: test + request: + payload: + allowed_actions: [] + type: index + description: Test action group + response: + status: 201 + - synopsis: Get action groups bulk. + path: /_plugins/_security/api/actiongroups + method: GET + response: + status: 200 + - synopsis: Patch action groups bulk. + path: /_plugins/_security/api/actiongroups + method: PATCH + request: + payload: + - op: add + path: /test + value: + allowed_actions: ['indices:admin/create', 'indices:admin/mapping/put'] + response: + status: 200 + - synopsis: Get action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: GET + parameters: + action_group: test + response: + status: 200 + - synopsis: Patch action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: PATCH + parameters: + action_group: test + request: + payload: + - op: replace + path: /allowed_actions + value: ['indices:admin/create', 'indices:admin/mapping/get'] + response: + status: 200 + - synopsis: Delete action group. + path: /_plugins/_security/api/actiongroups/{action_group} + method: DELETE + parameters: + action_group: test + response: + status: 200 diff --git a/tests/default/security/api/allowlist.yaml b/tests/default/security/api/allowlist.yaml new file mode 100644 index 000000000..6808131d8 --- /dev/null +++ b/tests/default/security/api/allowlist.yaml @@ -0,0 +1,31 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test allowlist endpoints. +version: '> 2.0' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Create an allowlist. + path: /_plugins/_security/api/allowlist + method: PUT + request: + payload: + enabled: true + requests: + /_cat/nodes: [GET] + response: + status: 403 + - synopsis: Get an allowlist. + path: /_plugins/_security/api/allowlist + method: GET + response: + status: 403 + - synopsis: Patch an allowlist. + path: /_plugins/_security/api/allowlist + method: PATCH + request: + payload: + - op: remove + path: /config/requests + response: + status: 403 diff --git a/tests/default/security/api/audit.yaml b/tests/default/security/api/audit.yaml new file mode 100644 index 000000000..7c004b24b --- /dev/null +++ b/tests/default/security/api/audit.yaml @@ -0,0 +1,20 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test audit endpoints. + +chapters: + - synopsis: Get an audit config. + path: /_plugins/_security/api/audit + method: GET + response: + status: 200 + - synopsis: Patch an audit config. + path: /_plugins/_security/api/audit + method: PATCH + request: + payload: + - op: add + path: /config/enabled + value: true + response: + status: 200 diff --git a/tests/default/security/api/audit/config.yaml b/tests/default/security/api/audit/config.yaml new file mode 100644 index 000000000..97335168c --- /dev/null +++ b/tests/default/security/api/audit/config.yaml @@ -0,0 +1,39 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test audit/config endpoint. + +chapters: + - synopsis: Create an audit config. + path: /_plugins/_security/api/audit/config + method: PUT + request: + payload: + enabled: true + audit: + ignore_users: [] + ignore_requests: [] + disabled_rest_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + disabled_transport_categories: + - AUTHENTICATED + - GRANTED_PRIVILEGES + log_request_body: false + resolve_indices: false + resolve_bulk_requests: false + exclude_sensitive_headers: true + enable_transport: false + enable_rest: true + compliance: + enabled: true + write_log_diffs: false + read_watched_fields: {} + read_ignore_users: [] + write_watched_indices: [] + write_ignore_users: [] + read_metadata_only: true + write_metadata_only: true + external_config: false + internal_config: true + response: + status: 200 diff --git a/tests/default/security/api/authtoken.yaml b/tests/default/security/api/authtoken.yaml new file mode 100644 index 000000000..4da5718b2 --- /dev/null +++ b/tests/default/security/api/authtoken.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test authtoken endpoint. + +chapters: + - synopsis: Create an auth token. + path: /_plugins/_security/api/authtoken + method: POST + response: + status: 200 diff --git a/tests/default/security/api/cache.yaml b/tests/default/security/api/cache.yaml new file mode 100644 index 000000000..d1f1d6d26 --- /dev/null +++ b/tests/default/security/api/cache.yaml @@ -0,0 +1,25 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test cache endpoint. + +chapters: + - synopsis: Get cache. + path: /_plugins/_security/api/cache + method: GET + response: + status: 501 + - synopsis: Create cache. + path: /_plugins/_security/api/cache + method: POST + response: + status: 501 + - synopsis: Update cache. + path: /_plugins/_security/api/cache + method: PUT + response: + status: 501 + - synopsis: Flush cache. + path: /_plugins/_security/api/cache + method: DELETE + response: + status: 200 diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml new file mode 100644 index 000000000..f4f23e817 --- /dev/null +++ b/tests/default/security/api/certificates.yaml @@ -0,0 +1,35 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test certificates endpoints. +version: '> 2.14' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get node. + id: get_node + path: /_cat/nodes + method: GET + parameters: + h: + - id + full_id: true + response: + status: 200 + content_type: text/plain + output: + node_id: payload + - synopsis: Get all certificates. + path: /_plugins/_security/api/certificates + method: GET + parameters: + cert_type: all + response: + status: 403 + - synopsis: Get node certificates. + path: /_plugins/_security/api/certificates/{node_id} + method: GET + parameters: + node_id: ${get_node.node_id} + cert_type: all + response: + status: 403 diff --git a/tests/default/security/api/generateonbehalfoftoken.yaml b/tests/default/security/api/generateonbehalfoftoken.yaml new file mode 100644 index 000000000..41b5f1453 --- /dev/null +++ b/tests/default/security/api/generateonbehalfoftoken.yaml @@ -0,0 +1,18 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test generateonbehalfoftoken endpoint. +version: '> 2.11' + +chapters: + - synopsis: Create an On-Behalf-Of token. + # Feature is disabled by default. https://opensearch.org/docs/latest/security/access-control/authentication-tokens/#configuration + path: /_plugins/_security/api/generateonbehalfoftoken + method: POST + request: + payload: + description: Auth token for admin + service: '' + duration: '60' + response: + content_type: text/plain + status: 400 diff --git a/tests/default/security/api/internalusers.yaml b/tests/default/security/api/internalusers.yaml new file mode 100644 index 000000000..784db4fcd --- /dev/null +++ b/tests/default/security/api/internalusers.yaml @@ -0,0 +1,60 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test internalusers endpoints. + +chapters: + - synopsis: Get internal users bulk. + path: /_plugins/_security/api/internalusers + method: GET + response: + status: 200 + - synopsis: Patch internal users bulk. + path: /_plugins/_security/api/internalusers + method: PATCH + request: + payload: + - op: add + path: /test + value: + backend_roles: [admin] + response: + status: 200 + - synopsis: Create internal user. + path: /_plugins/_security/api/internalusers/{username} + method: PUT + parameters: + username: test + request: + payload: + password: myWeakPassword123! + opendistro_security_roles: [] + backend_roles: [] + attributes: {} + response: + status: 200 + - synopsis: Get internal user. + path: /_plugins/_security/api/internalusers/{username} + method: GET + parameters: + username: test + response: + status: 200 + - synopsis: Patch internal user. + path: /_plugins/_security/api/internalusers/{username} + method: PATCH + parameters: + username: test + request: + payload: + - op: add + path: /opendistro_security_roles + value: [all_access] + response: + status: 200 + - synopsis: Delete internal user. + path: /_plugins/_security/api/internalusers/{username} + method: DELETE + parameters: + username: test + response: + status: 200 diff --git a/tests/default/security/api/internalusers.yml/authtoken.yml b/tests/default/security/api/internalusers.yml/authtoken.yml new file mode 100644 index 000000000..2cfb3c6a1 --- /dev/null +++ b/tests/default/security/api/internalusers.yml/authtoken.yml @@ -0,0 +1,35 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test internalusers/authtoken endpoint. +version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 + +prologues: + - path: /_plugins/_security/api/internalusers/{username} + method: PUT + parameters: + username: test + request: + payload: + opendistro_security_roles: [] + backend_roles: [] + attributes: + service: true + enabled: true + status: [201] + +chapters: + # Auth-tokens can only be vended for service accounts. + - synopsis: Create internal user token. + path: /_plugins/_security/api/internalusers/{username}/authtoken + method: POST + parameters: + username: test + response: + status: 200 + +epilogues: + - path: /_plugins/_security/api/internalusers/{username} + method: DELETE + parameters: + username: test + status: [200] diff --git a/tests/default/security/api/migrate.yaml b/tests/default/security/api/migrate.yaml new file mode 100644 index 000000000..3cf053554 --- /dev/null +++ b/tests/default/security/api/migrate.yaml @@ -0,0 +1,11 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test migrate endpoint. + +# BAD_REQUEST. Can not migrate configuration because it was already migrated. +chapters: + - synopsis: Migrate v6 to v7 config. + path: /_plugins/_security/api/migrate + method: POST + response: + status: 400 diff --git a/tests/default/security/api/nodesdn.yaml b/tests/default/security/api/nodesdn.yaml new file mode 100644 index 000000000..0cfcd1b43 --- /dev/null +++ b/tests/default/security/api/nodesdn.yaml @@ -0,0 +1,61 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test nodesdn endpoints. + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +# The setting `plugins. security. nodes_dn_dynamic_config_enabled` must be enabled. +chapters: + - synopsis: Get distinguished names. + path: /_plugins/_security/api/nodesdn + method: GET + parameters: + show_all: true + response: + status: 400 + - synopsis: Patch distinguished names. + path: /_plugins/_security/api/nodesdn + method: PATCH + request: + payload: + - op: replace + path: /cluster1/nodes_dn/0 + value: [''] + response: + status: 400 + - synopsis: Create distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PUT + parameters: + cluster_name: test + request: + payload: + nodes_dn: + - CN=cluster3.example.com + response: + status: 400 + - synopsis: Get distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: GET + parameters: + cluster_name: test + response: + status: 400 + - synopsis: Patch distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PATCH + parameters: + cluster_name: test + request: + payload: + op: replace + path: /test/nodes_dn/0 + value: [CN=cluster2.example.com] + response: + status: 400 + - synopsis: Delete distinguished name. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: DELETE + parameters: + cluster_name: test + response: + status: 400 diff --git a/tests/default/security/api/permissionsinfo.yaml b/tests/default/security/api/permissionsinfo.yaml new file mode 100644 index 000000000..59e1f9062 --- /dev/null +++ b/tests/default/security/api/permissionsinfo.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test permissionsinfo endpoint. + +chapters: + - synopsis: Get evaluated permissions for currently logged in user. + path: /_plugins/_security/api/permissionsinfo + method: GET + response: + status: 200 diff --git a/tests/default/security/api/roles.yaml b/tests/default/security/api/roles.yaml new file mode 100644 index 000000000..a79ef6312 --- /dev/null +++ b/tests/default/security/api/roles.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test roles endpoints. + +chapters: + - synopsis: Create role. + path: /_plugins/_security/api/roles/{role} + method: PUT + parameters: + role: test + request: + payload: + cluster_permissions: + - cluster_composite_ops + - indices_monitor + index_permissions: + - index_patterns: + - 'movies*' + dls: '' + fls: [] + masked_fields: [] + allowed_actions: + - read + tenant_permissions: + - tenant_patterns: + - human_resources + allowed_actions: + - kibana_all_read + response: + status: 201 + - synopsis: Get roles bulk. + path: /_plugins/_security/api/roles + method: GET + response: + status: 200 + - synopsis: Patch roles bulk. + path: /_plugins/_security/api/roles + method: PATCH + request: + payload: + - op: add + path: /test/index_permissions/0/fls + value: + - 'random*' + - ~random1 + response: + status: 200 + - synopsis: Get role. + path: /_plugins/_security/api/roles/{role} + method: GET + parameters: + role: test + response: + status: 200 + - synopsis: Patch role. + path: /_plugins/_security/api/roles/{role} + method: PATCH + parameters: + role: test + request: + payload: + - op: add + path: /index_permissions/0/fls + value: + - 'random*' + - ~random1 + response: + status: 200 + - synopsis: Delete role. + path: /_plugins/_security/api/roles/{role} + method: DELETE + parameters: + role: test + response: + status: 200 diff --git a/tests/default/security/api/rolesmapping.yaml b/tests/default/security/api/rolesmapping.yaml new file mode 100644 index 000000000..8fed57e18 --- /dev/null +++ b/tests/default/security/api/rolesmapping.yaml @@ -0,0 +1,77 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test rolesmapping endpoint. + +prologues: + - path: /_plugins/_security/api/roles/{role} + method: PUT + parameters: + role: test + request: + payload: + cluster_permissions: + - cluster_composite_ops + status: [201] +chapters: + - synopsis: Get rolesmapping bulk. + path: /_plugins/_security/api/rolesmapping + method: GET + response: + status: 200 + - synopsis: Patch rolesmapping bulk. + path: /_plugins/_security/api/rolesmapping + method: PATCH + request: + payload: + - op: add + path: /all_access + value: + users: [test] + backend_roles: [admin] + response: + status: 200 + - synopsis: Create rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: PUT + parameters: + role: test + request: + payload: + backend_roles: [captains] + hosts: + - '*.example.com' + users: [test] + response: + status: 201 + - synopsis: Get rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: GET + parameters: + role: test + response: + status: 200 + - synopsis: Patch rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: PATCH + parameters: + role: test + request: + payload: + - op: replace + path: /backend_roles + value: [admin] + response: + status: 200 + - synopsis: Delete rolesmapping. + path: /_plugins/_security/api/rolesmapping/{role} + method: DELETE + parameters: + role: test + response: + status: 200 +epilogues: + - path: /_plugins/_security/api/roles/{role} + method: DELETE + parameters: + role: test + status: [200] diff --git a/tests/default/security/api/securityconfig.yaml b/tests/default/security/api/securityconfig.yaml new file mode 100644 index 000000000..28c6573bd --- /dev/null +++ b/tests/default/security/api/securityconfig.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test securityconfig endpoints. +version: '> 2.9' + +# ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get a security config. + path: /_plugins/_security/api/securityconfig + method: GET + response: + status: 200 + - synopsis: Patch a security config. + path: /_plugins/_security/api/securityconfig + method: PATCH + request: + payload: + - op: replace + path: /config/dynamic/authc/basic_internal_auth_domain/transport_enabled + value: true + response: + status: 403 diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml new file mode 100644 index 000000000..5bae694be --- /dev/null +++ b/tests/default/security/api/securityconfig/config.yaml @@ -0,0 +1,43 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test securityconfig/config endpoint. +version: '>2.9' + +# ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. +chapters: + - synopsis: Update a security config. + path: /_plugins/_security/api/securityconfig/config + method: PUT + request: + payload: + dynamic: + filtered_alias_mode: warn + disable_rest_auth: false + disable_intertransport_auth: false + respect_request_indices_options: false + opensearch-dashboards: + multitenancy_enabled: true + server_username: kibanaserver + index: .opensearch-dashboards + http: + anonymous_auth_enabled: false + authc: + basic_internal_auth_domain: + http_enabled: true + transport_enabled: true + order: 0 + http_authenticator: + challenge: true + type: basic + config: {} + authentication_backend: + type: intern + config: {} + description: Authenticate via HTTP Basic against internal users database + auth_failure_listeners: {} + do_not_fail_on_forbidden: false + multi_rolespan_enabled: true + hosts_resolver_mode: ip-only + do_not_fail_on_forbidden_empty: false + response: + status: 403 diff --git a/tests/default/security/api/ssl/certs.yml b/tests/default/security/api/ssl/certs.yml new file mode 100644 index 000000000..168ff0964 --- /dev/null +++ b/tests/default/security/api/ssl/certs.yml @@ -0,0 +1,12 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/certs endpoint. +version: '>= 2.0' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Get ssl certificates. + path: /_plugins/_security/api/ssl/certs + method: GET + response: + status: 403 diff --git a/tests/default/security/api/ssl/http/reloadcerts.yaml b/tests/default/security/api/ssl/http/reloadcerts.yaml new file mode 100644 index 000000000..30f1f043b --- /dev/null +++ b/tests/default/security/api/ssl/http/reloadcerts.yaml @@ -0,0 +1,12 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/http/reloadcerts endpoint. +version: '> 2.7' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Reload http certs. + path: /_plugins/_security/api/ssl/http/reloadcerts + method: PUT + response: + status: 403 diff --git a/tests/default/security/api/ssl/transport/reloadcerts.yaml b/tests/default/security/api/ssl/transport/reloadcerts.yaml new file mode 100644 index 000000000..9585b1a17 --- /dev/null +++ b/tests/default/security/api/ssl/transport/reloadcerts.yaml @@ -0,0 +1,12 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: Test ssl/transport/reloadcerts endpoint. +version: '> 2.7' + +# ADMIN-CERT only. These tests require explicit rest api admin privileges. +chapters: + - synopsis: Reload transport certs. + path: /_plugins/_security/api/ssl/transport/reloadcerts + method: PUT + response: + status: 403 diff --git a/tests/default/security/api/tenancy/config.yaml b/tests/default/security/api/tenancy/config.yaml new file mode 100644 index 000000000..1e3d95262 --- /dev/null +++ b/tests/default/security/api/tenancy/config.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test tenancy/config endpoints. +version: '> 2.6' + +chapters: + - synopsis: Get tenancy config. + path: /_plugins/_security/api/tenancy/config + method: GET + response: + status: 200 + - synopsis: Create or Update tenancy config. + path: /_plugins/_security/api/tenancy/config + method: PUT + request: + payload: + default_tenant: admin_tenant + private_tenant_enabled: false + multitenancy_enabled: true + sign_in_options: [] + response: + status: 200 diff --git a/tests/default/security/api/tenants.yaml b/tests/default/security/api/tenants.yaml new file mode 100644 index 000000000..3e41617ee --- /dev/null +++ b/tests/default/security/api/tenants.yaml @@ -0,0 +1,56 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test tenants endpoints. + +chapters: + - synopsis: Create tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: PUT + parameters: + tenant: test + request: + payload: + description: A test tenant. + response: + status: 201 + - synopsis: Get tenants bulk. + path: /_plugins/_security/api/tenants + method: GET + response: + status: 200 + - synopsis: Patch tenants bulk. + path: /_plugins/_security/api/tenants + method: PATCH + request: + payload: + - op: replace + path: /test/description + value: A very good description + response: + status: 200 + - synopsis: Get tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: GET + parameters: + tenant: test + response: + status: 200 + - synopsis: Patch tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: PATCH + parameters: + tenant: test + request: + payload: + - op: replace + path: /description + value: An updated description + response: + status: 200 + - synopsis: Delete tenant. + path: /_plugins/_security/api/tenants/{tenant} + method: DELETE + parameters: + tenant: test + response: + status: 200 diff --git a/tests/default/security/api/user.yaml b/tests/default/security/api/user.yaml new file mode 100644 index 000000000..4c676f39a --- /dev/null +++ b/tests/default/security/api/user.yaml @@ -0,0 +1,37 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test user endpoints. + +chapters: + - synopsis: Get user bulk. + path: /_plugins/_security/api/user + method: GET + response: + status: 200 + - synopsis: Create user. + path: /_plugins/_security/api/user/{username} + method: PUT + parameters: + username: test + request: + payload: + password: myWeakPassword123! + opendistro_security_roles: [] + backend_roles: [] + attributes: {} + response: + status: 201 + - synopsis: Get user. + path: /_plugins/_security/api/user/{username} + method: GET + parameters: + username: test + response: + status: 200 + - synopsis: Delete user. + path: /_plugins/_security/api/user/{username} + method: DELETE + parameters: + username: test + response: + status: 200 diff --git a/tests/default/security/api/user/authtoken.yaml b/tests/default/security/api/user/authtoken.yaml new file mode 100644 index 000000000..edf450fb6 --- /dev/null +++ b/tests/default/security/api/user/authtoken.yaml @@ -0,0 +1,35 @@ +$schema: ../../../../../json_schemas/test_story.schema.yaml + +description: Test authtoken endpoints for user. +version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 + +prologues: + - path: /_plugins/_security/api/user/{username} + method: PUT + parameters: + username: test + request: + payload: + opendistro_security_roles: [] + backend_roles: [] + attributes: + service: true + enabled: true + status: [201] + +chapters: + # Auth-tokens can only be vended for service accounts. + - synopsis: Create user token. + path: /_plugins/_security/api/user/{username}/authtoken + method: POST + parameters: + username: test + response: + status: 501 + +epilogues: + - path: /_plugins/_security/api/user/{username} + method: DELETE + parameters: + username: test + status: [200] diff --git a/tests/default/security/api/validate.yaml b/tests/default/security/api/validate.yaml new file mode 100644 index 000000000..5c4cbdfc4 --- /dev/null +++ b/tests/default/security/api/validate.yaml @@ -0,0 +1,13 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test validate endpoint. + +# BAD_REQUEST. Can not migrate configuration because it was already migrated. +chapters: + - synopsis: Check whether v6 configuration is valid. + path: /_plugins/_security/api/validate + method: GET + parameters: + accept_invalid: false + response: + status: 400 diff --git a/tests/default/security/authinfo.yaml b/tests/default/security/authinfo.yaml new file mode 100644 index 000000000..1d65c31f0 --- /dev/null +++ b/tests/default/security/authinfo.yaml @@ -0,0 +1,34 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test authinfo endpoint. + +chapters: + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: < 2.13 + response: + status: 200 + - synopsis: Get auth info via POST. + path: /_plugins/_security/authinfo + method: POST + version: < 2.13 + response: + status: 200 + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: = 2.13 + parameters: + verbose: false + response: + status: 200 + - synopsis: Get auth info. + path: /_plugins/_security/authinfo + method: GET + version: '> 2.13' + parameters: + verbose: false + auth_type: basic + response: + status: 200 diff --git a/tests/default/security/dashboardsinfo.yaml b/tests/default/security/dashboardsinfo.yaml new file mode 100644 index 000000000..fb28a9914 --- /dev/null +++ b/tests/default/security/dashboardsinfo.yaml @@ -0,0 +1,15 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test dashboardsinfo endpoint. + +chapters: + - synopsis: Get dashboards info. + path: /_plugins/_security/dashboardsinfo + method: GET + response: + status: 200 + - synopsis: Get dashboards info via POST. + path: /_plugins/_security/dashboardsinfo + method: POST + response: + status: 200 diff --git a/tests/default/security/health.yaml b/tests/default/security/health.yaml new file mode 100644 index 000000000..b1c42b037 --- /dev/null +++ b/tests/default/security/health.yaml @@ -0,0 +1,27 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test health endpoint. + +chapters: + - synopsis: Get security health info. + path: /_plugins/_security/health + method: GET + parameters: + mode: strict + response: + status: 200 + payload: + message: null + mode: strict + status: UP + - synopsis: Get security health info via POST. + path: /_plugins/_security/health + method: POST + parameters: + mode: strict + response: + status: 200 + payload: + message: null + mode: strict + status: UP diff --git a/tests/default/security/sslinfo.yaml b/tests/default/security/sslinfo.yaml new file mode 100644 index 000000000..b73d18ee4 --- /dev/null +++ b/tests/default/security/sslinfo.yaml @@ -0,0 +1,25 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test sslinfo endpoint. + +chapters: + - synopsis: Get ssl info. + path: /_opendistro/_security/sslinfo + method: GET + parameters: + show_dn: false + response: + status: 200 + payload: + principal: null + peer_certificates: '0' + ssl_protocol: TLSv1.3 + ssl_openssl_available: false + ssl_openssl_version: -1 + ssl_openssl_version_string: null + ssl_openssl_non_available_cause: 'java.lang.ClassNotFoundException: io.netty.internal.tcnative.SSLContext' + ssl_openssl_supports_key_manager_factory: false + ssl_openssl_supports_hostname_validation: false + ssl_provider_http: JDK + ssl_provider_transport_server: JDK + ssl_provider_transport_client: JDK diff --git a/tests/default/security/tenantinfo.yaml b/tests/default/security/tenantinfo.yaml new file mode 100644 index 000000000..8b0aeee54 --- /dev/null +++ b/tests/default/security/tenantinfo.yaml @@ -0,0 +1,17 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test tenantinfo endpoint. + +chapters: + - synopsis: Get tenant info. + path: /_plugins/_security/tenantinfo + method: GET + response: + status: 403 # only allowed for super-admin or dashboards-server role mapping + content_type: text/plain + - synopsis: Get tenant info via POST. + path: /_plugins/_security/tenantinfo + method: POST + response: + status: 403 # only allowed for super-admin or dashboards-server role mapping + content_type: text/plain diff --git a/tests/default/security/whoami.yaml b/tests/default/security/whoami.yaml new file mode 100644 index 000000000..3e2c5016b --- /dev/null +++ b/tests/default/security/whoami.yaml @@ -0,0 +1,24 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test whoami endpoints. +version: '>= 2.0' + +chapters: + - synopsis: Get current user info. + path: /_plugins/_security/whoami + method: GET + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false + - synopsis: Get current user info via POST. + path: /_plugins/_security/whoami + method: POST + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false diff --git a/tests/default/security/whoamiprotected.yaml b/tests/default/security/whoamiprotected.yaml new file mode 100644 index 000000000..a3771fb7c --- /dev/null +++ b/tests/default/security/whoamiprotected.yaml @@ -0,0 +1,15 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test whoamiprotected endpoint. +version: '> 2.10' + +chapters: + - synopsis: Get current user info from protected endpoint. + path: /_plugins/_security/whoamiprotected + method: GET + response: + status: 200 + payload: + dn: null + is_admin: false + is_node_certificate_request: false From 08bbc0e2205c59b70f77620ce47a49333cc45f70 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 17:10:36 -0400 Subject: [PATCH 14/21] Fix: correctly named folders/files. (#497) * Fix: correctly named folders/files. Signed-off-by: dblock * Refresh index to avoid flaky test. Signed-off-by: dblock --------- Signed-off-by: dblock --- tests/default/indices/{mapping.yml => mapping.yaml} | 0 tests/default/observability/observability.yaml | 5 ++++- .../authtoken.yml => internalusers/authtoken.yaml} | 0 tests/default/security/api/ssl/{certs.yml => certs.yaml} | 0 .../security/api/{_upgrade_check.yaml => upgrade_check.yaml} | 0 .../api/{_upgrade_perform.yaml => upgrade_perform.yaml} | 0 6 files changed, 4 insertions(+), 1 deletion(-) rename tests/default/indices/{mapping.yml => mapping.yaml} (100%) rename tests/default/security/api/{internalusers.yml/authtoken.yml => internalusers/authtoken.yaml} (100%) rename tests/default/security/api/ssl/{certs.yml => certs.yaml} (100%) rename tests/default/security/api/{_upgrade_check.yaml => upgrade_check.yaml} (100%) rename tests/default/security/api/{_upgrade_perform.yaml => upgrade_perform.yaml} (100%) diff --git a/tests/default/indices/mapping.yml b/tests/default/indices/mapping.yaml similarity index 100% rename from tests/default/indices/mapping.yml rename to tests/default/indices/mapping.yaml diff --git a/tests/default/observability/observability.yaml b/tests/default/observability/observability.yaml index f9e37461c..e27b2be5e 100644 --- a/tests/default/observability/observability.yaml +++ b/tests/default/observability/observability.yaml @@ -57,6 +57,9 @@ prologues: - name: field1 type: text status: [200] + - path: /_refresh + method: POST + status: [200] chapters: - synopsis: Retrieve specific Observability object after creation. path: /_plugins/_observability/object/{object_id} @@ -144,7 +147,7 @@ chapters: status: 200 payload: startIndex: 0 - totalHits: 0 + totalHits: 1 totalHitRelation: eq observabilityObjectList: [] epilogues: diff --git a/tests/default/security/api/internalusers.yml/authtoken.yml b/tests/default/security/api/internalusers/authtoken.yaml similarity index 100% rename from tests/default/security/api/internalusers.yml/authtoken.yml rename to tests/default/security/api/internalusers/authtoken.yaml diff --git a/tests/default/security/api/ssl/certs.yml b/tests/default/security/api/ssl/certs.yaml similarity index 100% rename from tests/default/security/api/ssl/certs.yml rename to tests/default/security/api/ssl/certs.yaml diff --git a/tests/default/security/api/_upgrade_check.yaml b/tests/default/security/api/upgrade_check.yaml similarity index 100% rename from tests/default/security/api/_upgrade_check.yaml rename to tests/default/security/api/upgrade_check.yaml diff --git a/tests/default/security/api/_upgrade_perform.yaml b/tests/default/security/api/upgrade_perform.yaml similarity index 100% rename from tests/default/security/api/_upgrade_perform.yaml rename to tests/default/security/api/upgrade_perform.yaml From 2ee31047f57d3daa50cf1be72c75533683cb498f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 12 Aug 2024 17:31:46 -0400 Subject: [PATCH 15/21] Skip only known or hidden files. (#498) Signed-off-by: dblock --- tools/src/tester/TestRunner.ts | 7 +++---- tools/tests/tester/fixtures/stories/.ignore-dot-file | 1 + tools/tests/tester/fixtures/stories/docker-compose.yml | 1 + tools/tests/tester/fixtures/stories/ignore.wrong.extension | 1 - 4 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 tools/tests/tester/fixtures/stories/.ignore-dot-file create mode 100644 tools/tests/tester/fixtures/stories/docker-compose.yml delete mode 100644 tools/tests/tester/fixtures/stories/ignore.wrong.extension diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0c0f3c4ff..0b197d648 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -64,10 +64,9 @@ export default class TestRunner { #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { const path = file === '' ? folder : `${folder}/${file}` const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (fs.statSync(path).isFile()) { - if (!path.endsWith('.yaml')) { - return [] - } + if (file.startsWith('.') || file == 'docker-compose.yml') { + return [] + } else if (fs.statSync(path).isFile()) { const story: Story = read_yaml(path) return [{ display_path: next_prefix === '' ? basename(path) : next_prefix, diff --git a/tools/tests/tester/fixtures/stories/.ignore-dot-file b/tools/tests/tester/fixtures/stories/.ignore-dot-file new file mode 100644 index 000000000..11c2f4cfd --- /dev/null +++ b/tools/tests/tester/fixtures/stories/.ignore-dot-file @@ -0,0 +1 @@ +Hidden file, ignored. \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/docker-compose.yml b/tools/tests/tester/fixtures/stories/docker-compose.yml new file mode 100644 index 000000000..a47276c99 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/docker-compose.yml @@ -0,0 +1 @@ +# A docker-compose.yml file, ignored. diff --git a/tools/tests/tester/fixtures/stories/ignore.wrong.extension b/tools/tests/tester/fixtures/stories/ignore.wrong.extension deleted file mode 100644 index a649676ae..000000000 --- a/tools/tests/tester/fixtures/stories/ignore.wrong.extension +++ /dev/null @@ -1 +0,0 @@ -Not a .yaml file, should be ignored From 42fd4d7365320b29a14ea374da284066b07d5de1 Mon Sep 17 00:00:00 2001 From: Theo Nam Truong Date: Mon, 12 Aug 2024 16:08:26 -0600 Subject: [PATCH 16/21] Fixed schema refs for ml.yaml (#489) * Fixed schema refs for ml.yaml Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong * # Signed-off-by: Theo Truong --------- Signed-off-by: Theo Truong --- spec/namespaces/ml.yaml | 32 ++++++++++++++++++++++++-------- spec/schemas/ml._common.yaml | 5 +++-- tests/default/ml/models.yaml | 18 +++++++++++++----- 3 files changed, 40 insertions(+), 15 deletions(-) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 54b5ea532..2822ceb89 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -130,36 +130,52 @@ components: ml.search_models: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsQuery' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsQuery' responses: ml.register_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroupRegistration' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroupRegistration' ml.get_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' ml.delete_model_group@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' ml.register_model@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/Task' + schema: + type: object + properties: + status: + type: string + task_id: + type: string + required: + - status + - task_id ml.delete_model@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/ModelGroup' + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' ml.get_task@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/Task' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/Task' ml.search_models@200: content: application/json: - $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsResponse' + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/SearchModelsResponse' parameters: ml.get_model_group::path.model_group_id: name: model_group_id diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 3e06e2612..8d1b40d3e 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -108,10 +108,10 @@ components: properties: model_id: type: string - description: The model ID. + task_id: + type: string state: type: string - description: The state. enum: - CANCELLED - COMPLETED @@ -124,6 +124,7 @@ components: description: Task type. enum: - DEPLOY_MODEL + - REGISTER_MODEL function_name: type: string worker_node: diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index ac386c0a4..4bdac64e1 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -12,8 +12,8 @@ prologues: ml_commons: only_run_on_ml_node: false chapters: - - synopsis: Create model. - id: create_model + - synopsis: Register model. + id: register_model path: /_plugins/_ml/models/_register method: POST request: @@ -25,17 +25,25 @@ chapters: status: 200 output: task_id: payload.task_id - - synopsis: Wait for task. + - synopsis: Wait to get completed task. + id: get_completed_task path: /_plugins/_ml/tasks/{task_id} method: GET warnings: multiple-paths-detected: false parameters: - task_id: ${create_model.task_id} + task_id: ${register_model.task_id} response: status: 200 payload: state: COMPLETED + output: + model_id: payload.model_id retry: - count: 3 + count: 5 wait: 30000 + - synopsis: Delete model. + path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_task.model_id} + method: DELETE From 09824d0d0913a27ed9deeedf71fa4fffb1bed8ff Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 10:58:38 -0400 Subject: [PATCH 17/21] Add support for ML neural search. (#504) Signed-off-by: dblock --- .cspell | 6 +- CHANGELOG.md | 3 +- spec/namespaces/ml.yaml | 52 +++++ spec/schemas/_common.mapping.yaml | 36 ++++ spec/schemas/_common.query_dsl.yaml | 28 +++ spec/schemas/ml._common.yaml | 14 ++ .../ingest/pipeline/neural_search.yaml | 194 ++++++++++++++++++ .../text_embedding.yaml} | 2 +- 8 files changed, 331 insertions(+), 4 deletions(-) create mode 100644 tests/default/ingest/pipeline/neural_search.yaml rename tests/default/ingest/{pipeline.yaml => pipeline/text_embedding.yaml} (94%) diff --git a/.cspell b/.cspell index 1fcd3199c..65a801cbf 100644 --- a/.cspell +++ b/.cspell @@ -59,6 +59,7 @@ gsub Gsub haasephonetik heteroscedastic +hnsw homoscedastic hotthreads huggingface @@ -74,6 +75,7 @@ kstem kuromoji Kuromoji languageset +localstats Lovins lucene Lucene @@ -176,6 +178,7 @@ tokenfilters translog Translog tubone +Undeploys unigrams Unmanaged unmatch @@ -186,5 +189,4 @@ urldecode vectory whoamiprotected wordnet -Yrtsd -localstats \ No newline at end of file +Yrtsd \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 25a085e8a..e1f7db7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,7 +69,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471)) - Added support for running tests against Amazon OpenSearch ([#476](https://github.com/opensearch-project/opensearch-api-specification/pull/476)) - Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) -- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) ### Changed diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 2822ceb89..00054b4cb 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -53,6 +53,26 @@ paths: responses: '200': $ref: '#/components/responses/ml.delete_model@200' + /_plugins/_ml/models/{model_id}/_deploy: + post: + operationId: ml.deploy_model.0 + x-operation-group: ml.deploy_model + description: Deploys a model. + parameters: + - $ref: '#/components/parameters/ml.deploy_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.deploy_model@200' + /_plugins/_ml/models/{model_id}/_undeploy: + post: + operationId: ml.undeploy_model.0 + x-operation-group: ml.undeploy_model + description: Undeploys a model. + parameters: + - $ref: '#/components/parameters/ml.undeploy_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.undeploy_model@200' /_plugins/_ml/tasks/{task_id}: get: operationId: ml.get_task.0 @@ -161,6 +181,26 @@ components: required: - status - task_id + ml.deploy_model@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + task_id: + type: string + task_type: + type: string + required: + - status + - task_id + ml.undeploy_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/UndeployModelResponse' ml.delete_model@200: content: application/json: @@ -195,6 +235,18 @@ components: required: true schema: type: string + ml.deploy_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string + ml.undeploy_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.get_task::path.task_id: name: task_id in: path diff --git a/spec/schemas/_common.mapping.yaml b/spec/schemas/_common.mapping.yaml index cc8e8de0a..ef7a31a31 100644 --- a/spec/schemas/_common.mapping.yaml +++ b/spec/schemas/_common.mapping.yaml @@ -208,6 +208,7 @@ components: - $ref: '#/components/schemas/IntegerRangeProperty' - $ref: '#/components/schemas/IpRangeProperty' - $ref: '#/components/schemas/LongRangeProperty' + - $ref: '#/components/schemas/KnnVectorProperty' BinaryProperty: allOf: - $ref: '#/components/schemas/DocValuesPropertyBase' @@ -1129,6 +1130,30 @@ components: type: boolean index: type: boolean + KnnVectorPropertyBase: + type: object + properties: + dimension: + type: number + method: + $ref: '#/components/schemas/KnnVectorMethod' + required: + - dimension + KnnVectorMethod: + type: object + properties: + name: + type: string + space_type: + type: string + engine: + type: string + parameters: + type: object + additionalProperties: + type: object + required: + - name DoubleRangeProperty: allOf: - $ref: '#/components/schemas/RangePropertyBase' @@ -1184,6 +1209,17 @@ components: - long_range required: - type + KnnVectorProperty: + allOf: + - $ref: '#/components/schemas/KnnVectorPropertyBase' + - type: object + properties: + type: + type: string + enum: + - knn_vector + required: + - type MatchType: type: string enum: diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index a0d547cd5..a4bb57b06 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -112,6 +112,8 @@ components: $ref: '#/components/schemas/MultiMatchQuery' nested: $ref: '#/components/schemas/NestedQuery' + neural: + $ref: '#/components/schemas/NeuralQuery' parent_id: $ref: '#/components/schemas/ParentIdQuery' percolate: @@ -1222,6 +1224,32 @@ components: required: - path - query + NeuralQuery: + allOf: + - $ref: '#/components/schemas/QueryBase' + - type: object + additionalProperties: + $ref: '#/components/schemas/NeuralQueryVectorField' + NeuralQueryVectorField: + type: object + properties: + query_text: + type: string + query_image: + type: string + format: binary + model_id: + type: string + k: + type: integer + min_score: + type: number + max_distance: + type: number + filter: + $ref: '#/components/schemas/QueryContainer' + required: + - model_id ParentIdQuery: allOf: - $ref: '#/components/schemas/QueryBase' diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 8d1b40d3e..400a92fd3 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -139,5 +139,19 @@ components: format: int64 is_async: type: boolean + error: + type: string required: - state + UndeployModelResponse: + type: object + additionalProperties: + $ref: '#/components/schemas/UndeployModelResponseModels' + UndeployModelResponseModels: + type: object + properties: + stats: + $ref: '#/components/schemas/UndeployModelResponseStats' + UndeployModelResponseStats: + type: object + additionalProperties: true diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml new file mode 100644 index 000000000..0d1d89a22 --- /dev/null +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -0,0 +1,194 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation a neural search ingest pipeline. +prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins: + ml_commons: + only_run_on_ml_node: false +epilogues: + - path: /_ingest/pipeline/movies_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] + - path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + status: [200, 404] + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] + - path: /_plugins/_ml/model_groups/{model_group_id} + method: DELETE + status: [200, 404] + parameters: + model_group_id: ${create_model_group.test_model_group_id} +version: '>= 2.11' +chapters: + - synopsis: Create model group. + id: create_model_group + path: /_plugins/_ml/model_groups/_register + method: POST + request: + payload: + name: NLP_Group + description: Model group for NLP models. + response: + status: 200 + output: + test_model_group_id: payload.model_group_id + - synopsis: Register model. + id: register_model + path: /_plugins/_ml/models/_register + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + response: + status: 200 + output: + task_id: payload.task_id + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Create ingest pipeline for text embedding. + path: /_ingest/pipeline/{id} + method: PUT + parameters: + id: movies_pipeline + request: + payload: + description: Extracts text from field and embeds it. + processors: + - text_embedding: + model_id: text-embedding-model + field_map: + text: passage_embedding + response: + status: 200 + payload: + acknowledged: true + - synopsis: Create an index using the pipeline. + path: /{index} + method: PUT + parameters: + index: movies + request: + payload: + settings: + index.knn: true + default_pipeline: movies_pipeline + mappings: + properties: + title: + type: text + year: + type: integer + passage_embedding: + type: knn_vector + dimension: 768 + method: + engine: lucene + space_type: l2 + name: hnsw + parameters: {} + response: + status: 200 + payload: + acknowledged: true + - synopsis: Ingest data. + path: /_bulk + method: POST + parameters: + refresh: 'true' + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {author: Nicolas Winding Refn, title: Drive, year: 1960} + response: + status: 200 + - synopsis: Search. + path: /{index}/_search + method: POST + parameters: + index: movies + request: + payload: + _source: + excludes: [passage_embedding] + query: + bool: + should: + - script_score: + query: + neural: + passage_embedding: + query_text: Money + model_id: ${get_completed_register_model_task.model_id} + k: 100 + script: + source: _score * 1.5 + - script_score: + query: + match: + title: Moneyball + script: + source: _score * 1.7 + - synopsis: Undeploy a model. + path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + response: + status: 200 diff --git a/tests/default/ingest/pipeline.yaml b/tests/default/ingest/pipeline/text_embedding.yaml similarity index 94% rename from tests/default/ingest/pipeline.yaml rename to tests/default/ingest/pipeline/text_embedding.yaml index 3145dcb62..6ba46e8e9 100644 --- a/tests/default/ingest/pipeline.yaml +++ b/tests/default/ingest/pipeline/text_embedding.yaml @@ -1,4 +1,4 @@ -$schema: ../../../json_schemas/test_story.schema.yaml +$schema: ../../../../json_schemas/test_story.schema.yaml description: Test the creation of an ingest pipeline with a text embedding processor. epilogues: From 9d3bc340ccd7d049e7d6e14a4aff2293780cb446 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 17:53:00 -0400 Subject: [PATCH 18/21] Remove support for comma-separated semver range. (#501) Signed-off-by: dblock --- TESTING_GUIDE.md | 6 +-- .../ingest/pipeline/text_embedding.yaml | 2 +- .../security/api/securityconfig/config.yaml | 2 +- tools/src/_utils/semver.ts | 20 ++++++++ tools/src/merger/OpenApiVersionExtractor.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 2 +- tools/tests/_utils/semver.test.ts | 51 +++++++++++++++++++ tools/tests/tester/fixtures/evals/passed.yaml | 19 +++++++ .../tests/tester/fixtures/stories/passed.yaml | 6 +++ 9 files changed, 104 insertions(+), 8 deletions(-) create mode 100644 tools/src/_utils/semver.ts create mode 100644 tools/tests/_utils/semver.test.ts diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 61f6c2828..1328cc863 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -195,11 +195,11 @@ You can also reuse output in payload expectations. See [tests/plugins/index_stat ### Managing Versions -It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows. +It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver or a semver range in your test stories or chapters as follows. ```yaml -- synopsis: Search with `phase_took` added in OpenSearch 2.12. - version: '>= 2.12' +- synopsis: Search with `phase_took` added in OpenSearch 2.12 and removed in version 3. + version: '>=2.12 <3' path: /{index}/_search parameters: index: movies diff --git a/tests/default/ingest/pipeline/text_embedding.yaml b/tests/default/ingest/pipeline/text_embedding.yaml index 6ba46e8e9..fe70aee21 100644 --- a/tests/default/ingest/pipeline/text_embedding.yaml +++ b/tests/default/ingest/pipeline/text_embedding.yaml @@ -5,7 +5,7 @@ epilogues: - path: /_ingest/pipeline/books_pipeline method: DELETE status: [200, 404] -version: '>= 2.11, < 3.0' # TODO: re-enable using a 3.0 build with the neural-search plugin +version: '>= 2.11' chapters: - synopsis: Create ingest pipeline for text embedding. path: /_ingest/pipeline/{id} diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml index 5bae694be..3f51b38c3 100644 --- a/tests/default/security/api/securityconfig/config.yaml +++ b/tests/default/security/api/securityconfig/config.yaml @@ -1,7 +1,7 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test securityconfig/config endpoint. -version: '>2.9' +version: '> 2.9' # ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. chapters: diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts new file mode 100644 index 000000000..52de7e899 --- /dev/null +++ b/tools/src/_utils/semver.ts @@ -0,0 +1,20 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from 'semver' + +export function coerce(version: string) : string { + return semver.coerce(version)?.toString() ?? version +} + +export function satisfies(version: string | semver.SemVer | undefined, range: string): boolean { + if (version === undefined || version === '') return true + if (semver.validRange(range) == null) throw `Invalid semver ${range}.` + return semver.satisfies(version, range) +} diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 5d4164b1c..3f2fb396b 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -11,7 +11,7 @@ import _, { extend, isEmpty } from 'lodash' import { delete_matching_keys, find_refs, write_yaml } from '../helpers' import { Logger } from '../Logger' import { type OpenAPIV3 } from 'openapi-types' -import semver from 'semver' +import * as semver from '../_utils/semver' // Extract a versioned API export default class OpenApiVersionExtractor { @@ -22,7 +22,7 @@ export default class OpenApiVersionExtractor { constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { this._source_spec = source_spec - this._target_version = semver.coerce(target_version)?.toString() ?? target_version + this._target_version = semver.coerce(target_version) this._logger = logger this._spec = undefined } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index fede2e397..9020968fa 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -14,7 +14,7 @@ import { overall_result } from './helpers' import { StoryOutputs } from './StoryOutputs' import SupplementalChapterEvaluator from './SupplementalChapterEvaluator' import { ChapterOutput } from './ChapterOutput' -import * as semver from 'semver' +import * as semver from '../_utils/semver' import _ from 'lodash' export default class StoryEvaluator { diff --git a/tools/tests/_utils/semver.test.ts b/tools/tests/_utils/semver.test.ts new file mode 100644 index 000000000..2454549b7 --- /dev/null +++ b/tools/tests/_utils/semver.test.ts @@ -0,0 +1,51 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import * as semver from "../../src/_utils/semver"; + +describe('coerce', () => { + it ('null', () => { + expect(semver.coerce('')).toEqual('') + expect(semver.coerce('1.2.3')).toEqual('1.2.3') + expect(semver.coerce('1.2')).toEqual('1.2.0') + expect(semver.coerce('1')).toEqual('1.0.0') + }) +}); + +describe('satisfies', () => { + it ('defaults', () => { + expect(semver.satisfies('', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('semver', () => { + expect(semver.satisfies(semver.coerce('2.17.0'), '>= 1.3 < 99.0')).toBe(true) + }) + + it ('~', () => { + expect(semver.satisfies('2.17.0', '~> 2.x')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 1.x')).toBe(false) + expect(semver.satisfies('2.17.0', '~> 2.17.0')).toBe(true) + expect(semver.satisfies('2.17.0', '~> 2.18')).toBe(false) + }) + + it ('> <', () => { + expect(semver.satisfies('2.17.0', '> 2.999.0')).toBe(false) + expect(semver.satisfies('2.17.0', '< 3.0')).toBe(true) + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('ranges', () => { + expect(semver.satisfies('2.17.0', '>= 1.3 < 99.0')).toBe(true) + }) + + it ('invalid', () => { + expect(() => { semver.satisfies('1.2.3', '>= 1, < 2') }).toThrow('Invalid semver >= 1, < 2.') + }) +}); diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index 12c1b2c32..abe502379 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -176,6 +176,25 @@ chapters: overall: result: SKIPPED message: Skipped because version 2.16.0 does not satisfy >= 2.999.0. + - title: This GET /_cat/health should run (>= 1.3, < 99.0). + overall: + result: PASSED + path: GET /_cat/health + request: + parameters: + format: + result: PASSED + request: + result: PASSED + response: + status: + result: PASSED + payload_body: + result: PASSED + payload_schema: + result: PASSED + output_values: + result: SKIPPED epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml index b4b08b963..a882666b3 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/passed.yaml @@ -89,3 +89,9 @@ chapters: path: /_cat/health parameters: format: json + - synopsis: This GET /_cat/health should run (>= 1.3, < 99.0). + version: '>= 1.3 < 99.0' + method: GET + path: /_cat/health + parameters: + format: json From 83eab434d08d5b623c8518ac50b88331d146a783 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 13 Aug 2024 20:09:24 -0400 Subject: [PATCH 19/21] Added search response processors. (#505) Signed-off-by: dblock --- .cspell | 4 +- CHANGELOG.md | 1 + spec/schemas/search_pipeline._common.yaml | 215 +++++++++++++++++- .../request_processor/filter_query.yaml | 71 ++++++ .../response_processor/rename_field.yaml | 62 +++++ .../pipeline/response_processor/sort.yaml | 75 ++++++ .../ingest/pipeline/neural_search.yaml | 2 +- 7 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 tests/default/_core/search/pipeline/request_processor/filter_query.yaml create mode 100644 tests/default/_core/search/pipeline/response_processor/rename_field.yaml create mode 100644 tests/default/_core/search/pipeline/response_processor/sort.yaml diff --git a/.cspell b/.cspell index 65a801cbf..3656f245c 100644 --- a/.cspell +++ b/.cspell @@ -91,7 +91,6 @@ mmapfs mmdb mokotoff Moneyball -Moneyball msearch msmarco mtermvectors @@ -138,6 +137,9 @@ Reindex relo reloadcerts remotestore +rerank +Rerank +Reranker rethrottle Rethrottle rolesmapping diff --git a/CHANGELOG.md b/CHANGELOG.md index e1f7db7db..f99a07a70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added API spec for security plugin ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271)) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) +- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) ### Changed diff --git a/spec/schemas/search_pipeline._common.yaml b/spec/schemas/search_pipeline._common.yaml index 94fd37470..ebb14ee9a 100644 --- a/spec/schemas/search_pipeline._common.yaml +++ b/spec/schemas/search_pipeline._common.yaml @@ -23,7 +23,7 @@ components: response_processors: type: array items: - $ref: '#/components/schemas/RequestProcessor' + $ref: '#/components/schemas/ResponseProcessor' phase_results_processors: type: array items: @@ -187,6 +187,219 @@ components: type: string required: - sample_factor + ResponseProcessor: + oneOf: + - type: object + title: personalize_search_ranking + properties: + personalize_search_ranking: + $ref: '#/components/schemas/PersonalizeSearchRankingResponseProcessor' + required: + - personalize_search_ranking + - type: object + title: retrieval_augmented_generation + properties: + retrieval_augmented_generation: + $ref: '#/components/schemas/RetrievalAugmentedGenerationResponseProcessor' + required: + - retrieval_augmented_generation + - type: object + title: rename_field + properties: + rename_field: + $ref: '#/components/schemas/RenameFieldResponseProcessor' + required: + - rename_field + - type: object + title: rerank + properties: + rerank: + $ref: '#/components/schemas/RerankResponseProcessor' + required: + - rerank + - type: object + title: collapse + properties: + collapse: + $ref: '#/components/schemas/CollapseResponseProcessor' + required: + - collapse + - type: object + title: truncate_hits + properties: + truncate_hits: + $ref: '#/components/schemas/TruncateHitsResponseProcessor' + required: + - truncate_hits + - type: object + title: sort + properties: + sort: + $ref: '#/components/schemas/SortResponseProcessor' + required: + - sort + - type: object + title: split + properties: + split: + $ref: '#/components/schemas/SplitResponseProcessor' + required: + - split + PersonalizeSearchRankingResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + campaign_arn: + type: string + recipe: + type: string + weight: + type: number + format: float + item_id_field: + type: string + iam_role_arn: + type: string + required: + - campaign_arn + - recipe + - weight + RetrievalAugmentedGenerationResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + model_id: + type: string + context_field_list: + type: array + items: + type: string + system_prompt: + type: string + user_instructions: + type: string + required: + - context_field_list + - model_id + RenameFieldResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + target_field: + type: string + required: + - field + - target_field + RerankContext: + type: object + properties: + document_fields: + type: array + items: + type: string + required: + - document_fields + MLOpenSearchReranker: + type: object + properties: + model_id: + type: string + required: + - model_id + RerankResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + ml_opensearch: + $ref: '#/components/schemas/MLOpenSearchReranker' + context: + $ref: '#/components/schemas/RerankContext' + CollapseResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + context_prefix: + type: string + required: + - field + TruncateHitsResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + target_size: + type: integer + format: int32 + context_prefix: + type: string + SortResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + order: + type: string + target_field: + type: string + required: + - field + SplitResponseProcessor: + type: object + properties: + tag: + type: string + description: + type: string + ignore_failure: + type: boolean + field: + type: string + separator: + type: string + preserve_trailing: + type: boolean + target_field: + type: string + required: + - field + - separator PhaseResultsProcessor: oneOf: - type: object diff --git a/tests/default/_core/search/pipeline/request_processor/filter_query.yaml b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml new file mode 100644 index 000000000..e2d2cfb79 --- /dev/null +++ b/tests/default/_core/search/pipeline/request_processor/filter_query.yaml @@ -0,0 +1,71 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /_bulk + method: POST + parameters: + refresh: 'true' + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +epilogues: + - path: /_search/pipeline/filter_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: filter_pipeline + request: + payload: + request_processors: + - filter_query: + tag: tag + description: This processor restricts searches to 20th century movies. + query: + range: + year: + lte: 2000 + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: filter_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: filter_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive + director: Nicolas Winding Refn + year: 1960 diff --git a/tests/default/_core/search/pipeline/response_processor/rename_field.yaml b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml new file mode 100644 index 000000000..388d1414a --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml @@ -0,0 +1,62 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.8' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + name: Drive + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - rename_field: + field: name + target_field: title + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + title: Drive diff --git a/tests/default/_core/search/pipeline/response_processor/sort.yaml b/tests/default/_core/search/pipeline/response_processor/sort.yaml new file mode 100644 index 000000000..837501f66 --- /dev/null +++ b/tests/default/_core/search/pipeline/response_processor/sort.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: |- + Test the creation of a search pipeline with a response processor. +version: '>= 2.16' +prologues: + - path: /movies/_doc/1 + method: POST + parameters: + refresh: true + request: + payload: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + status: [201] +epilogues: + - path: /_search/pipeline/names_pipeline + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: names_pipeline + request: + payload: + response_processors: + - sort: + field: names + order: asc + target_field: sorted_names + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: names_pipeline + response: + status: 200 + - synopsis: Search. + warnings: + multiple-paths-detected: false + path: /{index}/_search + method: GET + parameters: + index: movies + search_pipeline: names_pipeline + response: + status: 200 + payload: + hits: + total: + value: 1 + hits: + - _index: movies + _source: + names: + - Drive + # eslint-disable-next-line yml/sort-sequence-values + - '1984' + - Moneyball + sorted_names: + - '1984' + - Drive + - Moneyball \ No newline at end of file diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 0d1d89a22..2d8b41ec1 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -155,7 +155,7 @@ chapters: - {create: {_index: movies}} - {director: Bennett Miller, title: Moneyball, year: 2011} - {create: {_index: movies}} - - {author: Nicolas Winding Refn, title: Drive, year: 1960} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} response: status: 200 - synopsis: Search. From 89e383b5bbad32c2a6b002763a5d655cda4d9994 Mon Sep 17 00:00:00 2001 From: Naarcha-AWS <97990722+Naarcha-AWS@users.noreply.github.com> Date: Wed, 14 Aug 2024 06:50:50 -0500 Subject: [PATCH 20/21] Add resolve API test (#506) --- tests/default/indices/index.yaml | 3 +-- tests/default/indices/resolve.yaml | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 tests/default/indices/resolve.yaml diff --git a/tests/default/indices/index.yaml b/tests/default/indices/index.yaml index d30186c8a..5446c4735 100644 --- a/tests/default/indices/index.yaml +++ b/tests/default/indices/index.yaml @@ -84,5 +84,4 @@ chapters: version: '>= 2.0' parameters: index: books,games - cluster_manager_timeout: 10s - + cluster_manager_timeout: 10s diff --git a/tests/default/indices/resolve.yaml b/tests/default/indices/resolve.yaml new file mode 100644 index 000000000..7c0fb45ce --- /dev/null +++ b/tests/default/indices/resolve.yaml @@ -0,0 +1,20 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Tests to see if the specified index exits using the `_resolve` endpoint. +prologues: + - path: /movies + method: PUT +epilogues: + - path: /movies + method: DELETE + status: [200, 404] + +chapters: + - synopsis: See if index `movies` exists. It should. + path: /_resolve/index/{name} + method: GET + parameters: + name: movies + expand_wildcards: none + response: + status: 200 \ No newline at end of file From 7dee041660986c5eb4bc85c2d19af006730746e9 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 14 Aug 2024 08:03:46 -0400 Subject: [PATCH 21/21] Added support for testing multiple distributions. (#483) * Added support for testing multiple distributions. Signed-off-by: dblock * Added x-distributions-included and excluded. Signed-off-by: dblock * Move node fetching to a prologue. Signed-off-by: dblock * Undo semver satisfies changes. Signed-off-by: dblock * Move OPENSEARCH_DISTRIBUTION_OPTION. Signed-off-by: dblock --------- Signed-off-by: dblock --- CHANGELOG.md | 6 ++ DEVELOPER_GUIDE.md | 4 ++ TESTING_GUIDE.md | 55 ++++++++++++++++++- json_schemas/test_story.schema.yaml | 11 ++++ spec/namespaces/_core.yaml | 2 + spec/schemas/nodes._common.yaml | 18 +++++- spec/schemas/nodes.info.yaml | 17 +----- tests/default/_core/info.yaml | 3 + .../_core/search/rest_total_hits_as_int.yaml | 1 - tests/default/cat/health.yaml | 10 ++-- tests/default/cat/indices.yaml | 4 ++ tests/default/cat/nodeattrs.yaml | 2 + tests/default/indices/cache.yaml | 2 + tests/default/indices/dangling.yaml | 2 + tests/default/indices/forcemerge.yaml | 2 + tests/default/indices/segments.yaml | 3 +- tests/default/indices/settings.yaml | 4 ++ tests/default/ml/model_groups.yaml | 2 + tests/default/ml/models.yaml | 2 + tests/default/security/api/certificates.yaml | 22 ++++---- tools/src/OpenSearchHttpClient.ts | 1 + tools/src/_utils/semver.ts | 3 +- tools/src/linter/SchemasValidator.ts | 4 +- tools/src/linter/components/OperationGroup.ts | 2 +- tools/src/merger/OpenApiVersionExtractor.ts | 38 +++++++++++-- tools/src/tester/MergedOpenApiSpec.ts | 8 ++- tools/src/tester/SchemaValidator.ts | 4 +- tools/src/tester/StoryEvaluator.ts | 21 +++++-- tools/src/tester/TestRunner.ts | 16 ++++-- tools/src/tester/test.ts | 7 ++- tools/src/tester/types/story.types.ts | 15 ++++- tools/src/types.ts | 2 + .../merger/OpenApiVersionExtractor.test.ts | 12 ++-- .../fixtures/extractor/expected_1.3.yaml | 6 ++ .../fixtures/extractor/expected_2.0.yaml | 6 ++ tools/tests/tester/MergedOpenApiSpec.test.ts | 53 +++++++++++++++--- .../fixtures/evals/skipped/distributions.yaml | 6 ++ .../specs/complete/namespaces/index.yaml | 18 ++++++ .../stories/skipped/distributions.yaml | 9 +++ tools/tests/tester/helpers.ts | 2 +- tools/tests/tester/integ/TestRunner.test.ts | 7 ++- 41 files changed, 332 insertions(+), 80 deletions(-) create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index f99a07a70..b8fd3cefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,11 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504)) - Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505)) +- Added `/_plugins/_security/api/certificates/` to API spec ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Changed @@ -116,6 +121,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479)) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) +- Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) ### Security diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 951f00cdd..43c30d7f3 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -146,6 +146,10 @@ This repository includes several OpenAPI Specification Extensions to fill in any - `x-ignorable`: Denotes that the operation should be ignored by the client generator. This is used in operation groups where some operations have been replaced by newer ones, but we still keep them in the specs because the server still supports them. - `x-global`: Denotes that the parameter is a global parameter that is included in every operation. These parameters are listed in the [spec/_global_parameters.yaml](spec/_global_parameters.yaml). - `x-default`: Contains the default value of a parameter. This is often used to override the default value specified in the schema, or to avoid accidentally changing the default value when updating a shared schema. +- `x-distributions-included`: Contains a list of distributions known to include the API. +- `x-distributions-excluded`: Contains a list of distributions known to exclude the API. + +Use `opensearch.org` for the official distribution in `x-distributions-*`, `amazon-managed` for Amazon Managed OpenSearch, and `amazon-serverless` for Amazon OpenSearch Serverless. ## Writing Spec Tests diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 1328cc863..d35c96aa5 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -11,6 +11,7 @@ - [Simple Test Story](#simple-test-story) - [Using Output from Previous Chapters](#using-output-from-previous-chapters) - [Managing Versions](#managing-versions) + - [Managing Distributions](#managing-distributions) - [Waiting for Tasks](#waiting-for-tasks) - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) @@ -209,7 +210,59 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne status: 200 ``` -The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. +The test tool will fetch the server version when it starts and use it automatically. The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests. + +### Managing Distributions + +OpenSearch consists of plugins that may or may not be present in various distributions. When adding a new API in the spec, you can specify `x-distributions-included` or `x-distributions-excluded` with a list of distributions that have a particular feature. For example, the Amazon Managed OpenSearch supports `GET /`, but Amazon Serverless OpenSearch does not. + +```yaml +/: + get: + operationId: info.0 + x-distributions-included: + - opensearch.org + - amazon-managed + x-distributions-excluded: + - amazon-serverless + description: Returns basic information about the cluster. +``` + +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. + +```yaml +description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org +chapters: + - synopsis: Get server info. + path: / + method: GET + response: + status: 200 +``` + +To test a particular distribution pass `--opensearch-distribution` to the test tool. For example, the following runs tests against an Amazon Managed OpenSearch instance. + +```bash +export AWS_ACCESS_KEY_ID=... +export AWS_SECRET_ACCESS_KEY=... +export AWS_SESSION_TOKEN=... +export AWS_REGION=us-west-2 + +export OPENSEARCH_URL=https://....us-west-2.es.amazonaws.com + +npm run test:spec -- --opensearch-distribution=amazon-managed +``` + +The output will visible skip APIs that are not available in the `amazon-managed` distribution. + +``` +PASSED _core/bulk.yaml (.../_core/bulk.yaml) +PASSED _core/info.yaml (.../_core/info.yaml) +SKIPPED indices/forcemerge.yaml (Skipped because distribution amazon-managed is not opensearch.org.) +``` ### Waiting for Tasks diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 6d1916509..928ba4bad 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -21,6 +21,8 @@ properties: $ref: '#/definitions/Chapter' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' required: [chapters,description] additionalProperties: false @@ -85,6 +87,8 @@ definitions: $ref: '#/definitions/Output' version: $ref: '#/definitions/Version' + distributions: + $ref: '#/definitions/Distributions' retry: $ref: '#/definitions/Retry' required: [method, path] @@ -106,6 +110,13 @@ definitions: The semver range to execute the story or chapter against. type: string + Distributions: + description: | + The list of distributions that support this API. + type: array + items: + type: string + Retry: description: | Number of times to retry on error. diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index 9aa5dbeaa..bc90411bb 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -9,6 +9,8 @@ paths: operationId: info.0 x-operation-group: info x-version-added: '1.0' + x-distributions-excluded: + - amazon-serverless description: Returns basic information about the cluster. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/schemas/nodes._common.yaml b/spec/schemas/nodes._common.yaml index ff0efdc21..cab8696c2 100644 --- a/spec/schemas/nodes._common.yaml +++ b/spec/schemas/nodes._common.yaml @@ -562,6 +562,14 @@ components: write_operations: description: The total number of write operations for the device completed since starting OpenSearch. type: number + read_time: + type: number + write_time: + type: number + queue_size: + type: number + io_time_in_millis: + $ref: '_common.yaml#/components/schemas/DurationValueUnitMillis' Jvm: type: object properties: @@ -1068,8 +1076,10 @@ components: type: boolean enforced: type: boolean + total_rejections_breakup: + $ref: '#/components/schemas/TotalRejectionsBreakup' total_rejections_breakup_shadow_mode: - $ref: '#/components/schemas/TotalRejectionsBreakupShadowMode' + $ref: '#/components/schemas/TotalRejectionsBreakup' ShardSearchBackpressureStats: type: object properties: @@ -1140,7 +1150,11 @@ components: type: number cancellation_limit_reached_count: type: number - TotalRejectionsBreakupShadowMode: + cancelled_task_percentage: + type: number + current_cancellation_eligible_tasks_count: + type: number + TotalRejectionsBreakup: type: object properties: node_limits: diff --git a/spec/schemas/nodes.info.yaml b/spec/schemas/nodes.info.yaml index 7086c17ca..6fbcefdc2 100644 --- a/spec/schemas/nodes.info.yaml +++ b/spec/schemas/nodes.info.yaml @@ -102,14 +102,10 @@ components: search_pipelines: $ref: '#/components/schemas/NodeInfoSearchPipelines' required: - - attributes - build_hash - build_type - - host - - ip - name - roles - - transport_address - version NodeInfoHttp: type: object @@ -156,7 +152,7 @@ components: bundled_jdk: type: boolean using_bundled_jdk: - type: boolean + type: [boolean, 'null'] using_compressed_ordinary_object_pointers: oneOf: - type: boolean @@ -167,16 +163,9 @@ components: type: string required: - bundled_jdk - - gc_collectors - - input_arguments - mem - - memory_pools - pid - start_time_in_millis - - version - - vm_name - - vm_vendor - - vm_version NodeInfoJvmMemory: type: object properties: @@ -256,12 +245,8 @@ components: swap: $ref: '#/components/schemas/NodeInfoMemory' required: - - arch - available_processors - - name - - pretty_name - refresh_interval_in_millis - - version NodeInfoOSCPU: type: object properties: diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 67e245235..9cc644a6b 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -2,6 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. +distributions: + - amazon-managed + - opensearch.org chapters: - synopsis: Get server info. path: / diff --git a/tests/default/_core/search/rest_total_hits_as_int.yaml b/tests/default/_core/search/rest_total_hits_as_int.yaml index 8e7a04351..32f9b3c2d 100644 --- a/tests/default/_core/search/rest_total_hits_as_int.yaml +++ b/tests/default/_core/search/rest_total_hits_as_int.yaml @@ -124,4 +124,3 @@ chapters: title: Drive year: 2011 order: 2 - diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 38865e9e5..196cff4b3 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -53,7 +53,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat with master response (format=json). method: GET @@ -66,7 +65,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_master: 'true' - synopsis: Cat with cluster_manager response (format=json). @@ -80,7 +78,6 @@ chapters: content_type: application/json payload: - node.total: '1' - status: yellow node.data: '1' discovered_cluster_manager: 'true' - synopsis: Cat in different formats (format=yaml). @@ -93,9 +90,10 @@ chapters: content_type: application/yaml payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -105,9 +103,10 @@ chapters: content_type: application/cbor payload: - node.total: '1' - status: yellow node.data: '1' - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/health parameters: @@ -117,5 +116,4 @@ chapters: content_type: application/smile payload: - node.total: '1' - status: yellow node.data: '1' diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index c20df6fe8..6b6130823 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -71,6 +71,8 @@ chapters: status: 200 content_type: application/yaml - synopsis: Cat in different formats (format=cbor). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: @@ -79,6 +81,8 @@ chapters: status: 200 content_type: application/cbor - synopsis: Cat in different formats (format=smile). + distributions: + - opensearch.org method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index cb9dc328c..01dba1cae 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -3,6 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. chapters: - synopsis: Cat with a json response. + distributions: + - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index db4db9dd5..2e056a526 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 94087d494..0b1b0c784 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. +distributions: + - opensearch.org chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index d537e3a7c..cc1608cb6 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -2,6 +2,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index da87fec7b..0445ebd64 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -1,7 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. - +distributions: + - opensearch.org prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 5bfada4b9..80906a7e3 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -11,6 +11,8 @@ epilogues: status: [200, 404] chapters: - synopsis: Get global settings. + distributions: + - opensearch.org path: /_settings method: GET parameters: @@ -23,6 +25,8 @@ chapters: response: status: 200 - synopsis: Get global settings (cluster_manager_timeout). + distributions: + - opensearch.org path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index a08efa0ce..8c036ae6e 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index 4bdac64e1..f92ee82d0 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -1,6 +1,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. +distributions: + - opensearch.org version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml index f4f23e817..340a659d5 100644 --- a/tests/default/security/api/certificates.yaml +++ b/tests/default/security/api/certificates.yaml @@ -4,20 +4,18 @@ description: Test certificates endpoints. version: '> 2.14' # ADMIN-CERT only. These tests require explicit rest api admin privileges. -chapters: - - synopsis: Get node. - id: get_node - path: /_cat/nodes +prologues: + - path: /_cat/nodes + id: node method: GET parameters: - h: - - id full_id: true - response: - status: 200 - content_type: text/plain + size: 1 + format: json + h: id output: - node_id: payload + id: payload[0].id +chapters: - synopsis: Get all certificates. path: /_plugins/_security/api/certificates method: GET @@ -29,7 +27,7 @@ chapters: path: /_plugins/_security/api/certificates/{node_id} method: GET parameters: - node_id: ${get_node.node_id} + node_id: ${node.id} cert_type: all response: - status: 403 + status: 403 \ No newline at end of file diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index afbb7d5fa..314a73d45 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -73,6 +73,7 @@ export interface OpenSearchHttpClientOptions { export type OpenSearchHttpClientCliOptions = { opensearchUrl?: string + opensearchDistribution?: string, opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean diff --git a/tools/src/_utils/semver.ts b/tools/src/_utils/semver.ts index 52de7e899..836c802ea 100644 --- a/tools/src/_utils/semver.ts +++ b/tools/src/_utils/semver.ts @@ -9,7 +9,8 @@ import * as semver from 'semver' -export function coerce(version: string) : string { +export function coerce(version?: string): undefined | string { + if (version === undefined) return undefined return semver.coerce(version)?.toString() ?? version } diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts index 06672285b..c463e49d9 100644 --- a/tools/src/linter/SchemasValidator.ts +++ b/tools/src/linter/SchemasValidator.ts @@ -16,7 +16,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemasValidator { diff --git a/tools/src/linter/components/OperationGroup.ts b/tools/src/linter/components/OperationGroup.ts index dc45765bc..f980e56b5 100644 --- a/tools/src/linter/components/OperationGroup.ts +++ b/tools/src/linter/components/OperationGroup.ts @@ -13,7 +13,7 @@ import ValidatorBase from './base/ValidatorBase' export default class OperationGroup extends ValidatorBase { static readonly OP_PRIORITY = ['operationId', 'x-operation-group', 'x-ignorable', 'deprecated', - 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', + 'x-deprecation-message', 'x-version-added', 'x-version-deprecated', 'x-version-removed', 'x-distributions-included', 'x-distributions-excluded', 'description', 'externalDocs', 'parameters', 'requestBody', 'responses'] name: string diff --git a/tools/src/merger/OpenApiVersionExtractor.ts b/tools/src/merger/OpenApiVersionExtractor.ts index 3f2fb396b..a2ccbffa4 100644 --- a/tools/src/merger/OpenApiVersionExtractor.ts +++ b/tools/src/merger/OpenApiVersionExtractor.ts @@ -17,12 +17,14 @@ import * as semver from '../_utils/semver' export default class OpenApiVersionExtractor { private _spec?: Record private _source_spec: OpenAPIV3.Document - private _target_version: string + private _target_version?: string + private _target_distribution?: string private _logger: Logger - constructor(source_spec: OpenAPIV3.Document, target_version: string, logger: Logger = new Logger()) { + constructor(source_spec: OpenAPIV3.Document, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this._source_spec = source_spec this._target_version = semver.coerce(target_version) + this._target_distribution = target_distribution this._logger = logger this._spec = undefined } @@ -45,16 +47,37 @@ export default class OpenApiVersionExtractor { #extract() : void { this._logger.info(`Extracting version ${this._target_version} ...`) this.#remove_keys_not_matching_semver() + this.#remove_keys_not_matching_distribution() this.#remove_unused() } #exclude_per_semver(obj: any): boolean { + if (this._target_version == undefined) return false + const x_version_added = semver.coerce(obj['x-version-added'] as string) const x_version_removed = semver.coerce(obj['x-version-removed'] as string) - if (x_version_added && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { + if (x_version_added !== undefined && !semver.satisfies(this._target_version, `>=${x_version_added.toString()}`)) { return true - } else if (x_version_removed && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + } else if (x_version_removed !== undefined && !semver.satisfies(this._target_version, `<${x_version_removed.toString()}`)) { + return true + } + + return false + } + + #exclude_per_distribution(obj: any): boolean { + if (this._target_distribution == undefined) return false + + const x_distributions_included = obj['x-distributions-included'] as string[] + + if (x_distributions_included?.length > 0 && !x_distributions_included.includes(this._target_distribution)) { + return true + } + + const x_distributions_excluded = obj['x-distributions-excluded'] as string[] + + if (x_distributions_excluded?.length > 0 && x_distributions_excluded.includes(this._target_distribution)) { return true } @@ -63,9 +86,16 @@ export default class OpenApiVersionExtractor { // Remove any elements that are x-version-added/removed incompatible with the target server version. #remove_keys_not_matching_semver(): void { + if (this._target_version == undefined) return delete_matching_keys(this._spec, this.#exclude_per_semver.bind(this)) } + // Remove any elements that are x-distributions-included incompatible with the target distribution. + #remove_keys_not_matching_distribution(): void { + if (this._target_distribution === undefined) return + delete_matching_keys(this._spec, this.#exclude_per_distribution.bind(this)) + } + #remove_unused(): void { if (this._spec === undefined) return diff --git a/tools/src/tester/MergedOpenApiSpec.ts b/tools/src/tester/MergedOpenApiSpec.ts index ee21f55ac..d4a3e2604 100644 --- a/tools/src/tester/MergedOpenApiSpec.ts +++ b/tools/src/tester/MergedOpenApiSpec.ts @@ -20,21 +20,23 @@ export default class MergedOpenApiSpec { logger: Logger file_path: string target_version?: string + target_distribution?: string protected _spec: OpenAPIV3.Document | undefined - constructor (spec_path: string, target_version?: string, logger: Logger = new Logger()) { + constructor (spec_path: string, target_version?: string, target_distribution?: string, logger: Logger = new Logger()) { this.logger = logger this.file_path = spec_path this.target_version = target_version + this.target_distribution = target_distribution } spec (): OpenAPIV3.Document { if (this._spec) return this._spec const merger = new OpenApiMerger(this.file_path, this.logger) var spec = merger.spec() - if (this.target_version !== undefined) { - const version_extractor = new OpenApiVersionExtractor(spec, this.target_version) + if (this.target_version !== undefined || this.target_distribution !== undefined) { + const version_extractor = new OpenApiVersionExtractor(spec, this.target_version, this.target_distribution) spec = version_extractor.extract() } const ctx = new SpecificationContext(this.file_path) diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 7f71e3870..01b55b5ad 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -19,7 +19,9 @@ const ADDITIONAL_KEYWORDS = [ 'x-version-added', 'x-version-deprecated', 'x-version-removed', - 'x-deprecation-message' + 'x-deprecation-message', + 'x-distributions-included', + 'x-distributions-excluded' ] export default class SchemaValidator { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 9020968fa..3f396a3c6 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -26,8 +26,8 @@ export default class StoryEvaluator { this._supplemental_chapter_evaluator = supplemental_chapter_evaluator } - async evaluate({ story, display_path, full_path }: StoryFile, version?: string, dry_run: boolean = false): Promise { - if (version != undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { + if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { return { result: Result.SKIPPED, display_path, @@ -37,13 +37,23 @@ export default class StoryEvaluator { } } + if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + } + } + const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path) if (variables_error !== undefined) { return variables_error } const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) - const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) return { display_path, @@ -76,7 +86,7 @@ export default class StoryEvaluator { } } - async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string): Promise { + async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { if (dry_run) { @@ -85,6 +95,9 @@ export default class StoryEvaluator { } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) + } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts index 0b197d648..dc880f946 100644 --- a/tools/src/tester/TestRunner.ts +++ b/tools/src/tester/TestRunner.ts @@ -34,19 +34,25 @@ export default class TestRunner { this._result_logger = result_logger } - async run (story_path: string, version?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { + async run (story_path: string, version?: string, distribution?: string, dry_run: boolean = false): Promise<{ results: StoryEvaluations, failed: boolean }> { let failed = false const story_files = this.story_files(story_path) const results: StoryEvaluations = { evaluations: [] } if (!dry_run) { - const info = await this._http_client.wait_until_available() - console.log(`OpenSearch ${ansi.green(info.version.number)}\n`) - version = info.version.number + if (distribution === 'amazon-serverless') { + // TODO: Fetch OpenSearch version when Amazon Serverless OpenSearch supports multiple. + version = '2.1' + } else { + const info = await this._http_client.wait_until_available() + version = info.version.number + } + + console.log(`OpenSearch ${ansi.green(version)}\n`) } for (const story_file of story_files) { - const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run) + const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run) results.evaluations.push(evaluation) this._result_logger.log(evaluation) if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 134888d7f..da025a34b 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -47,6 +47,9 @@ const command = new Command() .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) .addOption(new Option('--opensearch-version ', 'target OpenSearch schema version').default(undefined)) + .addOption(new Option('--opensearch-distribution ', 'OpenSearch distribution') + .default('opensearch.org') + .env('OPENSEARCH_DISTRIBUTION')) .addOption(OPENSEARCH_URL_OPTION) .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) @@ -63,7 +66,7 @@ const command = new Command() const opts = command.opts() const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) -const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Logger(LogLevel.error)) +const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, opts.opensearchDistribution, new Logger(LogLevel.error)) const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ responseType: 'arraybuffer', logger, ...opts })) const chapter_reader = new ChapterReader(http_client, logger) const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger) @@ -73,7 +76,7 @@ const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapt const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) const runner = new TestRunner(http_client, story_validator, story_evaluator, result_logger) -runner.run(opts.testsPath, spec.api_version(), opts.dryRun) +runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts.dryRun) .then( ({ results, failed }) => { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e418ff762..83d307b44 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -50,6 +50,14 @@ export type Payload = {} | any[] | string | number | boolean; * via the `definition` "Version". */ export type Version = string; +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export type Distributions = string[]; /** * Number of times to retry on error. * @@ -94,6 +102,7 @@ export interface Story { epilogues?: SupplementalChapter[]; chapters: Chapter[]; version?: Version; + distributions?: Distributions; } /** * This interface was referenced by `Story`'s JSON-Schema @@ -112,6 +121,7 @@ export interface ChapterRequest { request?: Request; output?: Output; version?: Version; + distributions?: Distributions; retry?: Retry; } /** @@ -131,7 +141,7 @@ export interface Request { * The values are paths to the values in the response. * The values should be in the form: * - `payload.` for the payload - * - `headers.` for the headers + * - `headers.` for the headers. * * * This interface was referenced by `Story`'s JSON-Schema @@ -157,6 +167,9 @@ export interface ExpectedResponse { * via the `definition` "Warnings". */ export interface Warnings { + /** + * Enable/disable warnings about multiple paths being tested in the same story. + */ 'multiple-paths-detected'?: boolean; } /** diff --git a/tools/src/types.ts b/tools/src/types.ts index a9497dbcb..b6871d022 100644 --- a/tools/src/types.ts +++ b/tools/src/types.ts @@ -16,6 +16,8 @@ export interface OperationSpec extends OpenAPIV3.OperationObject { 'x-version-deprecated'?: string 'x-deprecation-message'?: string 'x-ignorable'?: boolean + 'x-distributions-included'?: string[] + 'x-distributions-excluded'?: string[] parameters?: OpenAPIV3.ReferenceObject[] requestBody?: OpenAPIV3.ReferenceObject diff --git a/tools/tests/merger/OpenApiVersionExtractor.test.ts b/tools/tests/merger/OpenApiVersionExtractor.test.ts index 61bc42622..ca3e14eb0 100644 --- a/tools/tests/merger/OpenApiVersionExtractor.test.ts +++ b/tools/tests/merger/OpenApiVersionExtractor.test.ts @@ -17,7 +17,7 @@ describe('extract() from a merged API spec', () => { const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete') describe('1.3', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3') + const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3', 'ignore') describe('write_to', () => { var temp: tmp.DirResult @@ -43,18 +43,18 @@ describe('extract() from a merged API spec', () => { test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0', 'ignore') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', 'distributed-excluded-amazon-serverless' ]) }) @@ -81,12 +81,12 @@ describe('extract() from a merged API spec', () => { }) describe('2.1', () => { - const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1') + const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1', 'oracle-managed') test('has matching responses', () => { const spec = extractor.extract() expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml index d0fa222f7..a4497ccbb 100644 --- a/tools/tests/merger/fixtures/extractor/expected_1.3.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_1.3.yaml @@ -35,6 +35,10 @@ paths: x-version-removed: '2.0' added-1.3-removed-2.0: $ref: '#/components/responses/info@added-1.3-removed-2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -106,6 +110,8 @@ components: description: Added in 1.3, removed in 2.0 via attribute in response body. x-version-added: '1.3' x-version-removed: '2.0' + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@removed-2.0: description: Removed in 2.0 via attribute next to ref. nodes.info@200: diff --git a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml index cfb4279d2..c684b7af9 100644 --- a/tools/tests/merger/fixtures/extractor/expected_2.0.yaml +++ b/tools/tests/merger/fixtures/extractor/expected_2.0.yaml @@ -57,6 +57,10 @@ paths: added-2.0: $ref: '#/components/responses/info@added-2.0' x-version-added: '2.0' + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless parameters: [] /nodes: get: @@ -144,6 +148,8 @@ components: type: object info@added-2.0: description: Added in 2.0 via attribute next to ref. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. nodes.info@200: description: All nodes. content: diff --git a/tools/tests/tester/MergedOpenApiSpec.test.ts b/tools/tests/tester/MergedOpenApiSpec.test.ts index 7e6c64641..db852a030 100644 --- a/tools/tests/tester/MergedOpenApiSpec.test.ts +++ b/tools/tests/tester/MergedOpenApiSpec.test.ts @@ -13,7 +13,7 @@ import MergedOpenApiSpec from "tester/MergedOpenApiSpec" describe('merged API spec', () => { describe('defaults', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, undefined, new Logger()) test('has an api version', () => { expect(spec.api_version()).toEqual('1.2.3') @@ -30,7 +30,8 @@ describe('merged API spec', () => { test('has all responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1' + '200', '201', '404', '500','503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) @@ -65,31 +66,67 @@ describe('merged API spec', () => { }) describe('1.3', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '1.3', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0' + '200', '201', '404', '500', '503', 'removed-2.0', 'added-1.3-removed-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', undefined, 'oracle-managed', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.0', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', undefined, new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' + ]) + }) + }) + + describe('2.0 amazon-serverless', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'amazon-serverless', new Logger()) + + test('has matching responses', () => { + expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-included-all' + ]) + }) + }) + + describe('2.0 oracle-managed', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.0', 'oracle-managed', new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0' + '200', '201', '404', '500', '503', 'added-2.0', + 'distributed-excluded-amazon-serverless' ]) }) }) describe('2.1', () => { - const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', new Logger()) + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete', '2.1', undefined, new Logger()) test('has matching responses', () => { expect(_.keys(spec.spec().paths['/index']?.get?.responses)).toEqual([ - '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1' + '200', '201', '404', '500', '503', 'added-2.0', 'added-2.1', + 'distributed-included-all', 'distributed-included-amazon-managed', 'distributed-excluded-amazon-serverless' ]) }) }) diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml new file mode 100644 index 000000000..cc4ef9a19 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml + +result: SKIPPED +description: This story should be skipped because of distributions. +message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml index 68337dc67..0c0a09234 100644 --- a/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml +++ b/tools/tests/tester/fixtures/specs/complete/namespaces/index.yaml @@ -28,6 +28,20 @@ paths: $ref: '#/components/responses/info@500' '503': $ref: '#/components/responses/info@503' + distributed-included-all: + $ref: '#/components/responses/info@distributed-all' + x-distributions-included: + - amazon-managed + - amazon-serverless + - opensearch.org + distributed-included-amazon-managed: + $ref: '#/components/responses/info@distributed-amazon-managed' + x-distributions-included: + - amazon-managed + distributed-excluded-amazon-serverless: + $ref: '#/components/responses/info@distributed-all' + x-distributions-excluded: + - amazon-serverless components: responses: info@200: @@ -76,6 +90,10 @@ components: info@added-2.1: description: Added in 2.1 via attribute in response body. x-version-added: '2.1' + info@distributed-amazon-managed: + description: Distributed only in AOS. + info@distributed-all: + description: Distributed in opensearch.org, AOS and AOSS. info@500: content: application/json: diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml new file mode 100644 index 000000000..8efebbb2f --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions.yaml @@ -0,0 +1,9 @@ +$schema: ../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions. +distributions: + - another + - some +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index ce11b6ad8..d6d7e5639 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -141,5 +141,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s full_path, display_path: `${name}.yaml`, story: read_yaml(full_path) - }, process.env.OPENSEARCH_VERSION ?? '2.16.0')) + }, process.env.OPENSEARCH_VERSION ?? '2.16.0', process.env.OPENSEARCH_DISTRIBUTION ?? 'opensearch.org')) } diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index b0333504f..0259c29ac 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -16,7 +16,7 @@ test('stories folder', async () => { const info = await opensearch_http_client.wait_until_available() expect(info.version).toBeDefined() - const run = await test_runner.run('tools/tests/tester/fixtures/stories') + const run = await test_runner.run('tools/tests/tester/fixtures/stories', undefined, 'opensearch.org') expect(run.failed).toBeTruthy() @@ -29,14 +29,15 @@ test('stories folder', async () => { } const passed = load_expected_evaluation('passed', true) - const skipped = load_expected_evaluation('skipped/semver', true) + const skipped_semver = load_expected_evaluation('skipped/semver', true) + const skipped_distributions = load_expected_evaluation('skipped/distributions', true) const not_found = load_expected_evaluation('failed/not_found', true) const invalid_data = load_expected_evaluation('failed/invalid_data', true) const chapter_error = load_expected_evaluation('error/chapter_error', true) const output_error = load_expected_evaluation('error/output_error', true) const prologue_error = load_expected_evaluation('error/prologue_error', true) - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped] + const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] expect(actual_evaluations).toEqual(expected_evaluations) })