From df349272cae46d183f8ff72df557f1d4cc917f81 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 14 Aug 2024 17:22:42 -0400 Subject: [PATCH 1/8] Added distributions/included and excluded. (#510) * Added distributions/included and excluded. Signed-off-by: dblock * Test distribution before version. Signed-off-by: dblock * Passing npm run test:spec -- --opensearch-distribution=amazon-managed against Amazon OpenSearch 2.13. Signed-off-by: dblock --------- Signed-off-by: dblock --- TESTING_GUIDE.md | 9 +- json_schemas/test_story.schema.yaml | 10 + spec/namespaces/cat.yaml | 3 + spec/namespaces/indices.yaml | 15 ++ spec/namespaces/security.yaml | 246 ++++++++++++++++++ spec/schemas/security._common.yaml | 6 +- tests/default/_core/info.yaml | 4 +- tests/default/cat/health.yaml | 8 +- tests/default/cat/indices.yaml | 8 +- tests/default/cat/master.yaml | 2 +- tests/default/cat/nodeattrs.yaml | 6 +- tests/default/indices/cache.yaml | 4 +- tests/default/indices/dangling.yaml | 4 +- tests/default/indices/forcemerge.yaml | 4 +- tests/default/indices/segments.yaml | 4 +- tests/default/indices/settings.yaml | 8 +- .../ingest/pipeline/neural_search.yaml | 5 + tests/default/ml/model_groups.yaml | 4 +- tests/default/ml/models.yaml | 4 +- tests/default/query/datasources.yaml | 5 +- tests/default/security/api/account.yaml | 4 + tests/default/security/api/actiongroups.yaml | 4 + tests/default/security/api/allowlist.yaml | 4 + tests/default/security/api/audit.yaml | 4 + tests/default/security/api/audit/config.yaml | 5 +- tests/default/security/api/authtoken.yaml | 4 + tests/default/security/api/cache.yaml | 4 + tests/default/security/api/certificates.yaml | 4 + .../security/api/generateonbehalfoftoken.yaml | 4 + tests/default/security/api/internalusers.yaml | 4 + .../security/api/internalusers/authtoken.yaml | 4 + tests/default/security/api/migrate.yaml | 4 + tests/default/security/api/nodesdn.yaml | 4 + .../default/security/api/permissionsinfo.yaml | 4 + tests/default/security/api/roles.yaml | 4 + tests/default/security/api/rolesmapping.yaml | 4 + .../default/security/api/securityconfig.yaml | 4 + .../security/api/securityconfig/config.yaml | 4 + tests/default/security/api/ssl/certs.yaml | 4 + .../security/api/ssl/http/reloadcerts.yaml | 4 + .../api/ssl/transport/reloadcerts.yaml | 4 + .../default/security/api/tenancy/config.yaml | 4 + tests/default/security/api/tenants.yaml | 4 + tests/default/security/api/upgrade_check.yaml | 5 +- .../default/security/api/upgrade_perform.yaml | 5 +- tests/default/security/api/user.yaml | 4 + .../default/security/api/user/authtoken.yaml | 4 + tests/default/security/api/validate.yaml | 4 + tests/default/security/authinfo.yaml | 5 +- tests/default/security/dashboardsinfo.yaml | 5 +- tests/default/security/health.yaml | 5 +- tests/default/security/sslinfo.yaml | 3 - tests/default/security/tenantinfo.yaml | 5 +- tests/default/security/whoami.yaml | 5 +- tests/default/security/whoamiprotected.yaml | 5 +- tools/src/tester/ChapterEvaluator.ts | 5 +- tools/src/tester/StoryEvaluator.ts | 44 +++- .../tester/SupplementalChapterEvaluator.ts | 8 +- tools/src/tester/types/story.types.ts | 15 +- .../fixtures/evals/skipped/distributions.yaml | 6 - .../evals/skipped/distributions/chapters.yaml | 16 ++ .../evals/skipped/distributions/excluded.yaml | 6 + .../evals/skipped/distributions/included.yaml | 6 + .../skipped/distributions/chapters.yaml | 22 ++ .../skipped/distributions/excluded.yaml | 10 + .../included.yaml} | 8 +- tools/tests/tester/helpers.ts | 13 +- tools/tests/tester/integ/TestRunner.test.ts | 25 +- 68 files changed, 610 insertions(+), 81 deletions(-) delete mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml create mode 100644 tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml create mode 100644 tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml rename tools/tests/tester/fixtures/stories/skipped/{distributions.yaml => distributions/included.yaml} (51%) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index d35c96aa5..bda054ab7 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -228,13 +228,16 @@ OpenSearch consists of plugins that may or may not be present in various distrib description: Returns basic information about the cluster. ``` -Similarly, skip tests that are not applicable to a distribution by listing the distributions that support it. +Similarly, skip tests that are not applicable to a distribution by listing the distributions that support or do not support it. ```yaml description: Test root endpoint. distributions: - - amazon-managed - - opensearch.org + included: + - amazon-managed + - opensearch.org + excluded: + - amazon-serverless chapters: - synopsis: Get server info. path: / diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index 928ba4bad..b41a904ef 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -111,6 +111,16 @@ definitions: type: string Distributions: + description: | + The list of distributions that support this API. + type: object + properties: + included: + $ref: '#/definitions/DistributionsList' + excluded: + $ref: '#/definitions/DistributionsList' + + DistributionsList: description: | The list of distributions that support this API. type: array diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index b22637624..b58505898 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -291,6 +291,9 @@ paths: operationId: cat.nodeattrs.0 x-operation-group: cat.nodeattrs x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns information about custom node attributes. externalDocs: url: https://opensearch.org/docs/latest/api-reference/cat/cat-nodeattrs/ diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index 55f0c2999..d5e13e558 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -189,6 +189,9 @@ paths: operationId: indices.clear_cache.0 x-operation-group: indices.clear_cache x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Clears all or specific caches for one or more indices. externalDocs: url: https://opensearch.org/docs/latest/api-reference/index-apis/clear-index-cache/ @@ -319,6 +322,9 @@ paths: operationId: indices.forcemerge.0 x-operation-group: indices.forcemerge x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Performs the force merge operation on one or more indices. externalDocs: url: https://opensearch.org/docs/latest @@ -589,6 +595,9 @@ paths: operationId: indices.segments.0 x-operation-group: indices.segments x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Provides low-level information about segments in a Lucene index. externalDocs: url: https://opensearch.org/docs/latest @@ -605,6 +614,9 @@ paths: operationId: indices.get_settings.0 x-operation-group: indices.get_settings x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns settings for one or more indices. externalDocs: url: https://opensearch.org/docs/latest/api-reference/index-apis/get-settings/ @@ -1564,6 +1576,9 @@ paths: operationId: indices.segments.1 x-operation-group: indices.segments x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Provides low-level information about segments in a Lucene index. externalDocs: url: https://opensearch.org/docs/latest diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 72da8905e..5900516f5 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -22,6 +22,9 @@ paths: operationId: security.authinfo.0 x-operation-group: security.authinfo x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns the authentication information. parameters: - $ref: '#/components/parameters/security.authinfo::query.auth_type' @@ -35,6 +38,9 @@ paths: operationId: security.authinfo.1 x-operation-group: security.authinfo x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns the authentication information. parameters: - $ref: '#/components/parameters/security.authinfo::query.auth_type' @@ -49,6 +55,9 @@ paths: operationId: security.get_dashboards_info.0 x-operation-group: security.get_dashboards_info x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the current security-dashboards plugin configuration. responses: '200': @@ -59,6 +68,9 @@ paths: operationId: security.post_dashboards_info.1 x-operation-group: security.post_dashboards_info x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates the current security-dashboards plugin configuration. responses: '200': @@ -70,6 +82,9 @@ paths: operationId: security.health.0 x-operation-group: security.health x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Checks to see if the Security plugin is up and running. parameters: - $ref: '#/components/parameters/security.health::query.mode' @@ -82,6 +97,9 @@ paths: operationId: security.health.1 x-operation-group: security.health x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Checks to see if the Security plugin is up and running. parameters: - $ref: '#/components/parameters/security.health::query.mode' @@ -95,6 +113,9 @@ paths: operationId: security.tenant_info.0 x-operation-group: security.tenant_info x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the tenant names if any exist. Only accessible to super admins or kibanaserver user. responses: '200': @@ -107,6 +128,9 @@ paths: operationId: security.tenant_info.1 x-operation-group: security.tenant_info x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the tenant names if any exist. Only accessible to super admins or kibanaserver user. responses: '200': @@ -120,6 +144,9 @@ paths: operationId: security.who_am_i.0 x-operation-group: security.who_am_i x-version-added: '2.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Gets the user identity related information for currently logged in user. responses: '200': @@ -130,6 +157,9 @@ paths: operationId: security.who_am_i.1 x-operation-group: security.who_am_i x-version-added: '2.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Gets the user identity related information for currently logged in user. responses: '200': @@ -141,6 +171,9 @@ paths: operationId: security.who_am_i_protected.0 x-operation-group: security.who_am_i_protected x-version-added: '2.11' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Gets the user identity related information for currently logged in user. User needs to have access to this endpoint when authorization at REST layer is enabled. responses: '200': @@ -152,6 +185,9 @@ paths: operationId: security.config_upgrade_check.0 x-operation-group: security.config_upgrade_check x-version-added: '2.14' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Check whether or not an upgrade can be performed and what resources can be updated. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#configuration-upgrade-check @@ -163,6 +199,9 @@ paths: operationId: security.config_upgrade_perform.0 x-operation-group: security.config_upgrade_perform x-version-added: '2.14' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Helps cluster operator upgrade missing defaults and stale default definitions. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#configuration-upgrade @@ -178,6 +217,9 @@ paths: operationId: security.get_account_details.0 x-operation-group: security.get_account_details x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns account details for the current user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-account-details @@ -188,6 +230,9 @@ paths: operationId: security.change_password.0 x-operation-group: security.change_password x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Changes the password for the current user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#change-password @@ -203,6 +248,9 @@ paths: operationId: security.get_action_groups.0 x-operation-group: security.get_action_groups x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves all action groups. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-action-groups @@ -213,6 +261,9 @@ paths: operationId: security.patch_action_groups.1 x-operation-group: security.patch_action_groups x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates, updates, or deletes multiple action groups in a single call. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-action-groups @@ -226,6 +277,9 @@ paths: operationId: security.get_action_group.0 x-operation-group: security.get_action_group x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves one action group. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-action-group @@ -238,6 +292,9 @@ paths: operationId: security.create_action_group.0 x-operation-group: security.create_action_group x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified action group. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#create-action-group @@ -254,6 +311,9 @@ paths: operationId: security.patch_action_group.0 x-operation-group: security.patch_action_group x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates individual attributes of an action group. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-action-group @@ -268,6 +328,9 @@ paths: operationId: security.delete_action_group.0 x-operation-group: security.delete_action_group x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Delete a specified action group. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-action-group @@ -281,6 +344,9 @@ paths: operationId: security.get_allowlist.0 x-operation-group: security.get_allowlist x-version-added: '2.1' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -293,6 +359,9 @@ paths: operationId: security.create_allowlist.0 x-operation-group: security.create_allowlist x-version-added: '2.1' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the allowlisted APIs. Accessible via Super Admin certificate or REST API permission. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -307,6 +376,9 @@ paths: operationId: security.patch_allowlist.0 x-operation-group: security.patch_allowlist x-version-added: '2.1' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates the current list of allowed API accessible to normal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#access-control-for-the-api @@ -322,6 +394,9 @@ paths: operationId: security.get_audit_configuration.0 x-operation-group: security.get_audit_configuration x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the audit configuration. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#audit-logs @@ -332,6 +407,9 @@ paths: operationId: security.patch_audit_configuration.0 x-operation-group: security.patch_audit_configuration x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: A PATCH call is used to update specified fields in the audit configuration. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#audit-logs @@ -345,6 +423,9 @@ paths: operationId: security.update_audit_configuration.0 x-operation-group: security.update_audit_configuration x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates the audit configuration. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#audit-logs @@ -358,6 +439,9 @@ paths: operationId: security.authtoken.0 x-operation-group: security.authtoken x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns the authorization token. responses: '200': @@ -368,6 +452,9 @@ paths: x-operation-group: security.cache x-ignorable: true x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Not supported for cache API. responses: '501': @@ -376,6 +463,9 @@ paths: operationId: security.cache.2 x-operation-group: security.cache x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless x-ignorable: true description: Not supported for cache API. responses: @@ -385,6 +475,9 @@ paths: operationId: security.cache.3 x-operation-group: security.cache x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless x-ignorable: true description: Not supported for cache API. responses: @@ -394,6 +487,9 @@ paths: operationId: security.flush_cache.0 x-operation-group: security.flush_cache x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Flushes the Security plugin user, authentication, and authorization cache. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#flush-cache @@ -405,6 +501,9 @@ paths: operationId: security.get_all_certificates.0 x-operation-group: security.get_all_certificates x-version-added: '2.15' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the cluster security certificates. parameters: - $ref: '#/components/parameters/security.get_all_certificates::query.cert_type' @@ -421,6 +520,9 @@ paths: operationId: security.get_node_certificates.0 x-operation-group: security.get_node_certificates x-version-added: '2.15' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the given node's security certificates. parameters: - $ref: '#/components/parameters/security.get_node_certificates::path.node_id' @@ -438,6 +540,9 @@ paths: operationId: security.generate_obo_token.0 x-operation-group: security.generate_obo_token x-version-added: '2.12' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Generates On-Behalf-Of token for the current user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/authentication-tokens/#api-endpoint @@ -453,6 +558,9 @@ paths: operationId: security.get_users.0 x-operation-group: security.get_users x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieve all internal users. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-users @@ -463,6 +571,9 @@ paths: operationId: security.patch_users.0 x-operation-group: security.patch_users x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates, updates, or deletes multiple internal users in a single call. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-users @@ -476,6 +587,9 @@ paths: operationId: security.get_user.0 x-operation-group: security.get_user x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieve one internal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-user @@ -488,6 +602,9 @@ paths: operationId: security.create_user.0 x-operation-group: security.create_user x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#create-user @@ -502,6 +619,9 @@ paths: operationId: security.patch_user.0 x-operation-group: security.patch_user x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates individual attributes of an internal user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-user @@ -516,6 +636,9 @@ paths: operationId: security.delete_user.0 x-operation-group: security.delete_user x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Delete the specified user. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-user @@ -529,6 +652,9 @@ paths: operationId: security.generate_user_token.0 x-operation-group: security.generate_user_token x-version-added: '2.7' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Generates authorization token for the given user. parameters: - $ref: '#/components/parameters/security.generate_user_token::path.username' @@ -542,6 +668,9 @@ paths: operationId: security.migrate.0 x-operation-group: security.migrate x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Migrates security configuration from v6 to v7. responses: '200': @@ -553,6 +682,9 @@ paths: operationId: security.get_distinguished_names.0 x-operation-group: security.get_distinguished_names x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves distinguished names. Only accessible to super-admins and with rest-api permissions when enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-distinguished-names @@ -569,6 +701,9 @@ paths: operationId: security.patch_distinguished_names.0 x-operation-group: security.patch_distinguished_names x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Bulk update of distinguished names. Only accessible to super-admins and with rest-api permissions when enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#update-all-distinguished-names @@ -586,6 +721,9 @@ paths: operationId: security.get_distinguished_name.0 x-operation-group: security.get_distinguished_name x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves distinguished names. Only accessible to super-admins and with rest-api permissions when enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-distinguished-names @@ -603,6 +741,9 @@ paths: operationId: security.update_distinguished_name.0 x-operation-group: security.update_distinguished_name x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Adds or updates the specified distinguished names in the cluster or node allow list. Only accessible to super-admins and with rest-api permissions when enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#update-distinguished-names @@ -621,6 +762,9 @@ paths: operationId: security.patch_distinguished_name.0 x-operation-group: security.patch_distinguished_name x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates a distinguished cluster name for a specific cluster. Only accessible to super-admins and with rest-api permissions when enabled. parameters: - $ref: '#/components/parameters/security.patch_distinguished_name::path.cluster_name' @@ -637,6 +781,9 @@ paths: operationId: security.delete_distinguished_name.0 x-operation-group: security.delete_distinguished_name x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Deletes all distinguished names in the specified cluster or node allow list. Only accessible to super-admins and with rest-api permissions when enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-distinguished-names @@ -654,6 +801,9 @@ paths: operationId: security.get_permissions_info.0 x-operation-group: security.get_permissions_info x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Gets the evaluated REST API permissions for the currently logged in user. responses: '200': @@ -665,6 +815,9 @@ paths: operationId: security.get_roles.0 x-operation-group: security.get_roles x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves all roles. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-roles @@ -675,6 +828,9 @@ paths: operationId: security.patch_roles.0 x-operation-group: security.patch_roles x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates, updates, or deletes multiple roles in a single call. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-roles @@ -690,6 +846,9 @@ paths: operationId: security.get_role.0 x-operation-group: security.get_role x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves one role. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-role @@ -702,6 +861,9 @@ paths: operationId: security.create_role.0 x-operation-group: security.create_role x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified role. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#create-role @@ -718,6 +880,9 @@ paths: operationId: security.patch_role.0 x-operation-group: security.patch_role x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates individual attributes of a role. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-role @@ -734,6 +899,9 @@ paths: operationId: security.delete_role.0 x-operation-group: security.delete_role x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Delete the specified role. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-role @@ -747,6 +915,9 @@ paths: operationId: security.get_role_mappings.0 x-operation-group: security.get_role_mappings x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves all role mappings. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-role-mappings @@ -757,6 +928,9 @@ paths: operationId: security.patch_role_mappings.0 x-operation-group: security.patch_role_mappings x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or updates multiple role mappings in a single call. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-role-mappings @@ -772,6 +946,9 @@ paths: operationId: security.get_role_mapping.0 x-operation-group: security.get_role_mapping x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves one role mapping. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-role-mapping @@ -784,6 +961,9 @@ paths: operationId: security.create_role_mapping.0 x-operation-group: security.create_role_mapping x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified role mapping. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#create-role-mapping @@ -800,6 +980,9 @@ paths: operationId: security.patch_role_mapping.0 x-operation-group: security.patch_role_mapping x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Updates individual attributes of a role mapping. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-role-mapping @@ -816,6 +999,9 @@ paths: operationId: security.delete_role_mapping.0 x-operation-group: security.delete_role_mapping x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Deletes the specified role mapping. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-role-mapping @@ -829,6 +1015,9 @@ paths: operationId: security.get_configuration.0 x-operation-group: security.get_configuration x-version-added: '2.10' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Returns the current Security plugin configuration in JSON format. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-configuration @@ -839,6 +1028,9 @@ paths: operationId: security.patch_configuration.0 x-operation-group: security.patch_configuration x-version-added: '2.10' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: A PATCH call is used to update the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-configuration @@ -854,6 +1046,9 @@ paths: operationId: security.update_configuration.0 x-operation-group: security.update_configuration x-version-added: '2.10' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Adds or updates the existing configuration using the REST API. Only accessible by admins and users with rest api access and only when put or patch is enabled. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#update-configuration @@ -869,6 +1064,9 @@ paths: operationId: security.get_certificates.0 x-operation-group: security.get_certificates x-version-added: '2.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves the cluster security certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-certificates @@ -884,6 +1082,9 @@ paths: operationId: security.reload_http_certificates.0 x-operation-group: security.reload_http_certificates x-version-added: '2.8' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Reload HTTP layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-http-certificates @@ -899,6 +1100,9 @@ paths: operationId: security.reload_transport_certificates.0 x-operation-group: security.reload_transport_certificates x-version-added: '2.8' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Reload Transport layer communication certificates. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#reload-transport-certificates @@ -914,6 +1118,9 @@ paths: operationId: security.get_tenancy_config.0 x-operation-group: security.get_tenancy_config x-version-added: '2.7' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves multi-tenancy configuration. Only accessible to admins and users with REST API permissions. externalDocs: url: https://opensearch.org/docs/latest/security/multi-tenancy/dynamic-config/#configuring-multi-tenancy-with-the-rest-api @@ -926,6 +1133,9 @@ paths: operationId: security.create_update_tenancy_config.0 x-operation-group: security.create_update_tenancy_config x-version-added: '2.7' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the multi-tenancy configuration. Only accessible to admins and users with REST API permissions. externalDocs: url: https://opensearch.org/docs/latest/security/multi-tenancy/dynamic-config/#configuring-multi-tenancy-with-the-rest-api @@ -941,6 +1151,9 @@ paths: operationId: security.get_tenants.0 x-operation-group: security.get_tenants x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves all tenants. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-tenants @@ -953,6 +1166,9 @@ paths: operationId: security.patch_tenants.0 x-operation-group: security.patch_tenants x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Add, delete, or modify multiple tenants in a single call. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-tenants @@ -968,6 +1184,9 @@ paths: operationId: security.get_tenant.0 x-operation-group: security.get_tenant x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieves one tenant. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#get-tenant @@ -980,6 +1199,9 @@ paths: operationId: security.create_tenant.0 x-operation-group: security.create_tenant x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified tenant. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#create-tenant @@ -998,6 +1220,9 @@ paths: operationId: security.patch_tenant.0 x-operation-group: security.patch_tenant x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Add, delete, or modify a single tenant. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#patch-tenant @@ -1014,6 +1239,9 @@ paths: operationId: security.delete_tenant.0 x-operation-group: security.delete_tenant x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Delete the specified tenant. externalDocs: url: https://opensearch.org/docs/latest/security/access-control/api/#delete-action-group @@ -1029,6 +1257,9 @@ paths: operationId: security.get_users_legacy.0 x-operation-group: security.get_users_legacy x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieve all internal users. Legacy API. responses: '200': @@ -1038,6 +1269,9 @@ paths: operationId: security.get_user_legacy.0 x-operation-group: security.get_user_legacy x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Retrieve one user. Legacy API. parameters: - $ref: '#/components/parameters/security.get_user_legacy::path.username' @@ -1048,6 +1282,9 @@ paths: operationId: security.create_user_legacy.0 x-operation-group: security.create_user_legacy x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Creates or replaces the specified user. Legacy API. parameters: - $ref: '#/components/parameters/security.create_user_legacy::path.username' @@ -1062,6 +1299,9 @@ paths: operationId: security.delete_user_legacy.0 x-operation-group: security.delete_user_legacy x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Delete the specified user. Legacy API. parameters: - $ref: '#/components/parameters/security.delete_user_legacy::path.username' @@ -1073,6 +1313,9 @@ paths: operationId: security.generate_user_token_legacy.0 x-operation-group: security.generate_user_token_legacy x-version-added: '2.7' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Generates authorization token for the given user. Legacy API. Not Implemented. parameters: - $ref: '#/components/parameters/security.generate_user_token_legacy::path.username' @@ -1084,6 +1327,9 @@ paths: operationId: security.validate.0 x-operation-group: security.validate x-version-added: '1.0' + x-distributions-excluded: + - amazon-managed + - amazon-serverless description: Checks whether the v6 security configuration is valid and ready to be migrated to v7. parameters: - $ref: '#/components/parameters/security.validate::query.accept_invalid' diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index 36c976cb0..d98618758 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -622,10 +622,10 @@ components: items: type: string ssl_protocol: - type: string + type: ['null',string] description: Protocol for this ssl setup. ssl_cipher: - type: string + type: ['null',string] description: Cipher for this ssl setup. ssl_openssl_available: type: boolean @@ -646,7 +646,7 @@ components: type: boolean description: Indicates whether hostname validation is supported. ssl_provider_http: - type: string + type: ['null',string] description: Returns http provider's name. ssl_provider_transport_server: type: string diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index 9cc644a6b..f8a360360 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -3,8 +3,8 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test root endpoint. distributions: - - amazon-managed - - opensearch.org + excluded: + - amazon-serverless chapters: - synopsis: Get server info. path: / diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index 196cff4b3..dff956b3e 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -93,7 +93,9 @@ chapters: node.data: '1' - synopsis: Cat in different formats (format=cbor). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/health parameters: @@ -106,7 +108,9 @@ chapters: node.data: '1' - synopsis: Cat in different formats (format=smile). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/health parameters: diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index 6b6130823..2e2ed1711 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -72,7 +72,9 @@ chapters: content_type: application/yaml - synopsis: Cat in different formats (format=cbor). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/indices parameters: @@ -82,7 +84,9 @@ chapters: content_type: application/cbor - synopsis: Cat in different formats (format=smile). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless method: GET path: /_cat/indices parameters: diff --git a/tests/default/cat/master.yaml b/tests/default/cat/master.yaml index 435a27fa1..93ab62942 100644 --- a/tests/default/cat/master.yaml +++ b/tests/default/cat/master.yaml @@ -1,9 +1,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/master endpoints. +version: < 2.0 chapters: - synopsis: Cat with a json response. - version: < 2.0 path: /_cat/master method: GET parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index 01dba1cae..f59162b31 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -1,10 +1,12 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodeattrs endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Cat with a json response. - distributions: - - opensearch.org path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/indices/cache.yaml b/tests/default/indices/cache.yaml index 2e056a526..a079e25e5 100644 --- a/tests/default/indices/cache.yaml +++ b/tests/default/indices/cache.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test index clear cache. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/dangling.yaml b/tests/default/indices/dangling.yaml index 0b1b0c784..419338d91 100644 --- a/tests/default/indices/dangling.yaml +++ b/tests/default/indices/dangling.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dangling indexes. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get dangling indexes. path: /_dangling diff --git a/tests/default/indices/forcemerge.yaml b/tests/default/indices/forcemerge.yaml index cc1608cb6..8fb96e627 100644 --- a/tests/default/indices/forcemerge.yaml +++ b/tests/default/indices/forcemerge.yaml @@ -3,7 +3,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test force merging an index. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/segments.yaml b/tests/default/indices/segments.yaml index 0445ebd64..0a481f578 100644 --- a/tests/default/indices/segments.yaml +++ b/tests/default/indices/segments.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: This story tests the Segments API. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /movies method: PUT diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 80906a7e3..4c9700c1d 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -12,7 +12,9 @@ epilogues: chapters: - synopsis: Get global settings. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless path: /_settings method: GET parameters: @@ -26,7 +28,9 @@ chapters: status: 200 - synopsis: Get global settings (cluster_manager_timeout). distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless path: /_settings method: GET version: '>= 2.0' diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 2d8b41ec1..43d476e64 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -1,6 +1,11 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test the creation a neural search ingest pipeline. +distributions: + # requires an ML node to be provisioned, disabling with only_run_on_ml_node is not possible + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /_cluster/settings method: PUT diff --git a/tests/default/ml/model_groups.yaml b/tests/default/ml/model_groups.yaml index 8c036ae6e..33ad68cbe 100644 --- a/tests/default/ml/model_groups.yaml +++ b/tests/default/ml/model_groups.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of model groups. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/ml/models.yaml b/tests/default/ml/models.yaml index f92ee82d0..2e0a94524 100644 --- a/tests/default/ml/models.yaml +++ b/tests/default/ml/models.yaml @@ -2,7 +2,9 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test the creation of models. distributions: - - opensearch.org + excluded: + - amazon-managed + - amazon-serverless version: '>= 2.11' prologues: - path: /_cluster/settings diff --git a/tests/default/query/datasources.yaml b/tests/default/query/datasources.yaml index 35c9fea04..80ad12800 100644 --- a/tests/default/query/datasources.yaml +++ b/tests/default/query/datasources.yaml @@ -2,7 +2,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test various operations of the OpenSearch Query Datasources API. version: '>=2.7' - +distributions: + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /_plugins/_query/_datasources/{datasource_name} method: DELETE diff --git a/tests/default/security/api/account.yaml b/tests/default/security/api/account.yaml index 2fc10499f..2668aeddd 100644 --- a/tests/default/security/api/account.yaml +++ b/tests/default/security/api/account.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test account endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get account details. diff --git a/tests/default/security/api/actiongroups.yaml b/tests/default/security/api/actiongroups.yaml index 0321d9022..6b2dfff52 100644 --- a/tests/default/security/api/actiongroups.yaml +++ b/tests/default/security/api/actiongroups.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test actiongroups endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create action group. diff --git a/tests/default/security/api/allowlist.yaml b/tests/default/security/api/allowlist.yaml index 6808131d8..5c0b29a8e 100644 --- a/tests/default/security/api/allowlist.yaml +++ b/tests/default/security/api/allowlist.yaml @@ -2,6 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test allowlist endpoints. version: '> 2.0' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/audit.yaml b/tests/default/security/api/audit.yaml index 7c004b24b..094651f53 100644 --- a/tests/default/security/api/audit.yaml +++ b/tests/default/security/api/audit.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test audit endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get an audit config. diff --git a/tests/default/security/api/audit/config.yaml b/tests/default/security/api/audit/config.yaml index 97335168c..9a5ab5213 100644 --- a/tests/default/security/api/audit/config.yaml +++ b/tests/default/security/api/audit/config.yaml @@ -1,7 +1,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test audit/config endpoint. - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create an audit config. path: /_plugins/_security/api/audit/config diff --git a/tests/default/security/api/authtoken.yaml b/tests/default/security/api/authtoken.yaml index 4da5718b2..36ea369b0 100644 --- a/tests/default/security/api/authtoken.yaml +++ b/tests/default/security/api/authtoken.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test authtoken endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create an auth token. diff --git a/tests/default/security/api/cache.yaml b/tests/default/security/api/cache.yaml index d1f1d6d26..9535c5c8f 100644 --- a/tests/default/security/api/cache.yaml +++ b/tests/default/security/api/cache.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test cache endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get cache. diff --git a/tests/default/security/api/certificates.yaml b/tests/default/security/api/certificates.yaml index 340a659d5..a1e868157 100644 --- a/tests/default/security/api/certificates.yaml +++ b/tests/default/security/api/certificates.yaml @@ -2,6 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test certificates endpoints. version: '> 2.14' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. prologues: diff --git a/tests/default/security/api/generateonbehalfoftoken.yaml b/tests/default/security/api/generateonbehalfoftoken.yaml index 41b5f1453..561355e77 100644 --- a/tests/default/security/api/generateonbehalfoftoken.yaml +++ b/tests/default/security/api/generateonbehalfoftoken.yaml @@ -2,6 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test generateonbehalfoftoken endpoint. version: '> 2.11' +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create an On-Behalf-Of token. diff --git a/tests/default/security/api/internalusers.yaml b/tests/default/security/api/internalusers.yaml index 784db4fcd..9285ea3b2 100644 --- a/tests/default/security/api/internalusers.yaml +++ b/tests/default/security/api/internalusers.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test internalusers endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get internal users bulk. diff --git a/tests/default/security/api/internalusers/authtoken.yaml b/tests/default/security/api/internalusers/authtoken.yaml index 2cfb3c6a1..08f7a40d8 100644 --- a/tests/default/security/api/internalusers/authtoken.yaml +++ b/tests/default/security/api/internalusers/authtoken.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test internalusers/authtoken endpoint. version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 +distributions: + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /_plugins/_security/api/internalusers/{username} diff --git a/tests/default/security/api/migrate.yaml b/tests/default/security/api/migrate.yaml index 3cf053554..98f3d13cd 100644 --- a/tests/default/security/api/migrate.yaml +++ b/tests/default/security/api/migrate.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test migrate endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless # BAD_REQUEST. Can not migrate configuration because it was already migrated. chapters: diff --git a/tests/default/security/api/nodesdn.yaml b/tests/default/security/api/nodesdn.yaml index 0cfcd1b43..43b3991d4 100644 --- a/tests/default/security/api/nodesdn.yaml +++ b/tests/default/security/api/nodesdn.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test nodesdn endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. # The setting `plugins. security. nodes_dn_dynamic_config_enabled` must be enabled. diff --git a/tests/default/security/api/permissionsinfo.yaml b/tests/default/security/api/permissionsinfo.yaml index 59e1f9062..f4af2607e 100644 --- a/tests/default/security/api/permissionsinfo.yaml +++ b/tests/default/security/api/permissionsinfo.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test permissionsinfo endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get evaluated permissions for currently logged in user. diff --git a/tests/default/security/api/roles.yaml b/tests/default/security/api/roles.yaml index a79ef6312..9e21d2473 100644 --- a/tests/default/security/api/roles.yaml +++ b/tests/default/security/api/roles.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test roles endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create role. diff --git a/tests/default/security/api/rolesmapping.yaml b/tests/default/security/api/rolesmapping.yaml index 8fed57e18..f7d8644f1 100644 --- a/tests/default/security/api/rolesmapping.yaml +++ b/tests/default/security/api/rolesmapping.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test rolesmapping endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /_plugins/_security/api/roles/{role} diff --git a/tests/default/security/api/securityconfig.yaml b/tests/default/security/api/securityconfig.yaml index 28c6573bd..3c961343b 100644 --- a/tests/default/security/api/securityconfig.yaml +++ b/tests/default/security/api/securityconfig.yaml @@ -2,6 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test securityconfig endpoints. version: '> 2.9' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/securityconfig/config.yaml b/tests/default/security/api/securityconfig/config.yaml index 3f51b38c3..050fff9a9 100644 --- a/tests/default/security/api/securityconfig/config.yaml +++ b/tests/default/security/api/securityconfig/config.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test securityconfig/config endpoint. version: '> 2.9' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only (except GET). These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/ssl/certs.yaml b/tests/default/security/api/ssl/certs.yaml index 168ff0964..c888789cb 100644 --- a/tests/default/security/api/ssl/certs.yaml +++ b/tests/default/security/api/ssl/certs.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test ssl/certs endpoint. version: '>= 2.0' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/ssl/http/reloadcerts.yaml b/tests/default/security/api/ssl/http/reloadcerts.yaml index 30f1f043b..6c16242b0 100644 --- a/tests/default/security/api/ssl/http/reloadcerts.yaml +++ b/tests/default/security/api/ssl/http/reloadcerts.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../../json_schemas/test_story.schema.yaml description: Test ssl/http/reloadcerts endpoint. version: '> 2.7' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/ssl/transport/reloadcerts.yaml b/tests/default/security/api/ssl/transport/reloadcerts.yaml index 9585b1a17..41431e00b 100644 --- a/tests/default/security/api/ssl/transport/reloadcerts.yaml +++ b/tests/default/security/api/ssl/transport/reloadcerts.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../../json_schemas/test_story.schema.yaml description: Test ssl/transport/reloadcerts endpoint. version: '> 2.7' +distributions: + excluded: + - amazon-managed + - amazon-serverless # ADMIN-CERT only. These tests require explicit rest api admin privileges. chapters: diff --git a/tests/default/security/api/tenancy/config.yaml b/tests/default/security/api/tenancy/config.yaml index 1e3d95262..51bc7864f 100644 --- a/tests/default/security/api/tenancy/config.yaml +++ b/tests/default/security/api/tenancy/config.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test tenancy/config endpoints. version: '> 2.6' +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get tenancy config. diff --git a/tests/default/security/api/tenants.yaml b/tests/default/security/api/tenants.yaml index 3e41617ee..4a2c2ecdd 100644 --- a/tests/default/security/api/tenants.yaml +++ b/tests/default/security/api/tenants.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test tenants endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Create tenant. diff --git a/tests/default/security/api/upgrade_check.yaml b/tests/default/security/api/upgrade_check.yaml index 60d553753..8cb9c7789 100644 --- a/tests/default/security/api/upgrade_check.yaml +++ b/tests/default/security/api/upgrade_check.yaml @@ -2,7 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test _upgrade_check endpoint. version: '> 2.13' - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Check whether an upgrade can be performed. path: /_plugins/_security/api/_upgrade_check diff --git a/tests/default/security/api/upgrade_perform.yaml b/tests/default/security/api/upgrade_perform.yaml index 643f7896f..39e452f83 100644 --- a/tests/default/security/api/upgrade_perform.yaml +++ b/tests/default/security/api/upgrade_perform.yaml @@ -2,7 +2,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test _upgrade_perform endpoint. version: '> 2.13' - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Perform the upgrade. path: /_plugins/_security/api/_upgrade_perform diff --git a/tests/default/security/api/user.yaml b/tests/default/security/api/user.yaml index 4c676f39a..bc931e023 100644 --- a/tests/default/security/api/user.yaml +++ b/tests/default/security/api/user.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test user endpoints. +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get user bulk. diff --git a/tests/default/security/api/user/authtoken.yaml b/tests/default/security/api/user/authtoken.yaml index edf450fb6..8df0c3574 100644 --- a/tests/default/security/api/user/authtoken.yaml +++ b/tests/default/security/api/user/authtoken.yaml @@ -2,6 +2,10 @@ $schema: ../../../../../json_schemas/test_story.schema.yaml description: Test authtoken endpoints for user. version: '> 2.16' # Fixed via https://github.com/opensearch-project/security/pull/4628 +distributions: + excluded: + - amazon-managed + - amazon-serverless prologues: - path: /_plugins/_security/api/user/{username} diff --git a/tests/default/security/api/validate.yaml b/tests/default/security/api/validate.yaml index 5c4cbdfc4..55987ab40 100644 --- a/tests/default/security/api/validate.yaml +++ b/tests/default/security/api/validate.yaml @@ -1,6 +1,10 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test validate endpoint. +distributions: + excluded: + - amazon-managed + - amazon-serverless # BAD_REQUEST. Can not migrate configuration because it was already migrated. chapters: diff --git a/tests/default/security/authinfo.yaml b/tests/default/security/authinfo.yaml index 1d65c31f0..5243a3f6a 100644 --- a/tests/default/security/authinfo.yaml +++ b/tests/default/security/authinfo.yaml @@ -1,7 +1,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test authinfo endpoint. - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get auth info. path: /_plugins/_security/authinfo diff --git a/tests/default/security/dashboardsinfo.yaml b/tests/default/security/dashboardsinfo.yaml index fb28a9914..6bcfd4b44 100644 --- a/tests/default/security/dashboardsinfo.yaml +++ b/tests/default/security/dashboardsinfo.yaml @@ -1,7 +1,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test dashboardsinfo endpoint. - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get dashboards info. path: /_plugins/_security/dashboardsinfo diff --git a/tests/default/security/health.yaml b/tests/default/security/health.yaml index b1c42b037..cbd8e9c5a 100644 --- a/tests/default/security/health.yaml +++ b/tests/default/security/health.yaml @@ -1,7 +1,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test health endpoint. - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get security health info. path: /_plugins/_security/health diff --git a/tests/default/security/sslinfo.yaml b/tests/default/security/sslinfo.yaml index b73d18ee4..acb164fc1 100644 --- a/tests/default/security/sslinfo.yaml +++ b/tests/default/security/sslinfo.yaml @@ -1,7 +1,6 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test sslinfo endpoint. - chapters: - synopsis: Get ssl info. path: /_opendistro/_security/sslinfo @@ -13,13 +12,11 @@ chapters: payload: principal: null peer_certificates: '0' - ssl_protocol: TLSv1.3 ssl_openssl_available: false ssl_openssl_version: -1 ssl_openssl_version_string: null ssl_openssl_non_available_cause: 'java.lang.ClassNotFoundException: io.netty.internal.tcnative.SSLContext' ssl_openssl_supports_key_manager_factory: false ssl_openssl_supports_hostname_validation: false - ssl_provider_http: JDK ssl_provider_transport_server: JDK ssl_provider_transport_client: JDK diff --git a/tests/default/security/tenantinfo.yaml b/tests/default/security/tenantinfo.yaml index 8b0aeee54..c113a1de7 100644 --- a/tests/default/security/tenantinfo.yaml +++ b/tests/default/security/tenantinfo.yaml @@ -1,7 +1,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test tenantinfo endpoint. - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get tenant info. path: /_plugins/_security/tenantinfo diff --git a/tests/default/security/whoami.yaml b/tests/default/security/whoami.yaml index 3e2c5016b..dc6c42c37 100644 --- a/tests/default/security/whoami.yaml +++ b/tests/default/security/whoami.yaml @@ -2,7 +2,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test whoami endpoints. version: '>= 2.0' - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get current user info. path: /_plugins/_security/whoami diff --git a/tests/default/security/whoamiprotected.yaml b/tests/default/security/whoamiprotected.yaml index a3771fb7c..46409fbf1 100644 --- a/tests/default/security/whoamiprotected.yaml +++ b/tests/default/security/whoamiprotected.yaml @@ -2,7 +2,10 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test whoamiprotected endpoint. version: '> 2.10' - +distributions: + excluded: + - amazon-managed + - amazon-serverless chapters: - synopsis: Get current user info from protected endpoint. path: /_plugins/_security/whoamiprotected diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index ae73f90a3..dee177ddc 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -87,7 +87,6 @@ export default class ChapterEvaluator { path: `${chapter.method} ${chapter.path}`, overall: { result: overall_result(evaluations) }, request: { parameters: params, request }, - retries, response: { status, payload_body: payload_body_evaluation, @@ -96,6 +95,10 @@ export default class ChapterEvaluator { } } + if (retries !== undefined) { + result.retries = retries + } + if (output_values_evaluation?.output !== undefined) { result.output = output_values_evaluation?.output } diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 3f396a3c6..46371a503 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -27,23 +27,33 @@ export default class StoryEvaluator { } async evaluate({ story, display_path, full_path }: StoryFile, version?: string, distribution?: string, dry_run: boolean = false): Promise { - if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + if (distribution != undefined && story.distributions?.included !== undefined && story.distributions?.included.length > 0 && !story.distributions.included.includes(distribution)) { return { result: Result.SKIPPED, display_path, full_path, description: story.description, - message: `Skipped because version ${version} does not satisfy ${story.version}.` + message: `Skipped because distribution ${distribution} is not ${story.distributions.included.length > 1 ? 'one of ' : ''}${story.distributions.included.join(', ')}.` } } - if (distribution != undefined && story.distributions !== undefined && !story.distributions.includes(distribution)) { + if (distribution != undefined && story.distributions?.excluded !== undefined && story.distributions?.excluded.length > 0 && story.distributions.excluded.includes(distribution)) { return { result: Result.SKIPPED, display_path, full_path, description: story.description, - message: `Skipped because distribution ${distribution} is not ${story.distributions.length > 1 ? 'one of ' : ''}${story.distributions.join(', ')}.` + message: `Skipped because distribution ${distribution} is ${story.distributions.excluded.length > 1 ? 'one of ' : ''}${story.distributions.excluded.join(', ')}.` + } + } + + if (version !== undefined && story.version !== undefined && !semver.satisfies(version, story.version)) { + return { + result: Result.SKIPPED, + display_path, + full_path, + description: story.description, + message: `Skipped because version ${version} does not satisfy ${story.version}.` } } @@ -55,7 +65,8 @@ export default class StoryEvaluator { const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs) - return { + + const result: StoryEvaluation = { display_path, full_path, description: story.description, @@ -63,8 +74,14 @@ export default class StoryEvaluator { prologues, epilogues, result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)), - warnings: this.#chapter_warnings(story.chapters) } + + const warnings = this.#chapter_warnings(story.chapters) + if (warnings !== undefined) { + result.warnings = warnings + } + + return result } #chapter_warnings(chapters: Chapter[]): string[] | undefined { @@ -91,13 +108,16 @@ export default class StoryEvaluator { for (const chapter of chapters) { if (dry_run) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) - } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { + evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) + } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.`, error: undefined } }) - } else if (distribution != undefined && chapter.distributions !== undefined && !chapter.distributions.includes(distribution)) { + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.included.length > 1 ? 'one of ' : ''}${chapter.distributions.included.join(', ')}.` } }) + } else if (distribution != undefined && chapter.distributions?.excluded !== undefined && chapter.distributions?.excluded.length > 0 && chapter.distributions.excluded.includes(distribution)) { + const title = chapter.synopsis || `${chapter.method} ${chapter.path}` + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } }) + } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` - evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.length > 1 ? 'one of ' : ''}${chapter.distributions.join(', ')}.`, error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } }) } else { const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs) has_errors = has_errors || evaluation.overall.result === Result.ERROR @@ -116,7 +136,7 @@ export default class StoryEvaluator { for (const chapter of chapters) { const title = `${chapter.method} ${chapter.path}` if (dry_run) { - evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) + evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } }) } else { const { evaluation, evaluation_error } = await this._supplemental_chapter_evaluator.evaluate(chapter, story_outputs) has_errors = has_errors || evaluation_error diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts index e507e9915..faa6c1004 100644 --- a/tools/src/tester/SupplementalChapterEvaluator.ts +++ b/tools/src/tester/SupplementalChapterEvaluator.ts @@ -12,7 +12,7 @@ import { ChapterOutput } from "./ChapterOutput"; import ChapterReader from "./ChapterReader"; import { StoryOutputs } from "./StoryOutputs"; import { overall_result } from "./helpers"; -import { ChapterEvaluation, Result } from "./types/eval.types"; +import { ChapterEvaluation, Evaluation, Result } from "./types/eval.types"; import { SupplementalChapter } from "./types/story.types"; import { Logger } from "../Logger"; import { to_json } from "../helpers"; @@ -71,9 +71,11 @@ export default class SupplementalChapterEvaluator { const message = message_segments.join('\n') + var overall: Evaluation = { result: Result.ERROR, message } + if (response.error !== undefined) overall.error = response.error as Error + var evaluation: ChapterEvaluation = { - title, - overall: { result: Result.ERROR, message, error: response.error as Error } + title, overall } if (output_values_evaluation.output) { diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index 83d307b44..e1937b112 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -55,9 +55,9 @@ export type Version = string; * * * This interface was referenced by `Story`'s JSON-Schema - * via the `definition` "Distributions". + * via the `definition` "DistributionsList". */ -export type Distributions = string[]; +export type DistributionsList = string[]; /** * Number of times to retry on error. * @@ -150,6 +150,17 @@ export interface Request { export interface Output { [k: string]: string; } +/** + * The list of distributions that support this API. + * + * + * This interface was referenced by `Story`'s JSON-Schema + * via the `definition` "Distributions". + */ +export interface Distributions { + included?: DistributionsList; + excluded?: DistributionsList; +} /** * This interface was referenced by `Story`'s JSON-Schema * via the `definition` "ExpectedResponse". diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions.yaml deleted file mode 100644 index cc4ef9a19..000000000 --- a/tools/tests/tester/fixtures/evals/skipped/distributions.yaml +++ /dev/null @@ -1,6 +0,0 @@ -display_path: skipped/distributions.yaml -full_path: tools/tests/tester/fixtures/stories/skipped/distributions.yaml - -result: SKIPPED -description: This story should be skipped because of distributions. -message: Skipped because distribution opensearch.org is not one of another, some. diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml new file mode 100644 index 000000000..e392a9d81 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/chapters.yaml @@ -0,0 +1,16 @@ +display_path: skipped/distributions/chapters.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml + +description: This story has chapters with distributions. +prologues: [] +epilogues: [] +result: SKIPPED +chapters: + - title: This chapter is excluded because of excluded distributions (opensearch.org). + overall: + message: Skipped because distribution opensearch.org is opensearch.org. + result: SKIPPED + - title: This chapter is excluded because of included distributions (opensearch.org). + overall: + message: Skipped because distribution opensearch.org is not amazon-managed. + result: SKIPPED diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml new file mode 100644 index 000000000..b5ddfd547 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/excluded.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions/excluded.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml + +result: SKIPPED +description: This story should be skipped because of distributions (excluded). +message: Skipped because distribution opensearch.org is one of amazon-managed, opensearch.org. \ No newline at end of file diff --git a/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml b/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml new file mode 100644 index 000000000..391decc11 --- /dev/null +++ b/tools/tests/tester/fixtures/evals/skipped/distributions/included.yaml @@ -0,0 +1,6 @@ +display_path: skipped/distributions/included.yaml +full_path: tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml + +result: SKIPPED +description: This story should be skipped because of distributions (included). +message: Skipped because distribution opensearch.org is not amazon-managed. diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml new file mode 100644 index 000000000..527b40dfa --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/chapters.yaml @@ -0,0 +1,22 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story has chapters with distributions. +prologues: [] +epilogues: [] +chapters: + - synopsis: This chapter is excluded because of excluded distributions (opensearch.org). + distributions: + excluded: + - opensearch.org + path: /{index} + method: PUT + parameters: + index: index + - synopsis: This chapter is excluded because of included distributions (opensearch.org). + distributions: + included: + - amazon-managed + path: /{index} + method: PUT + parameters: + index: index diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml new file mode 100644 index 000000000..38102bc76 --- /dev/null +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/excluded.yaml @@ -0,0 +1,10 @@ +$schema: ../../../../../../../json_schemas/test_story.schema.yaml + +description: This story should be skipped because of distributions (excluded). +distributions: + excluded: + - amazon-managed + - opensearch.org +prologues: [] +epilogues: [] +chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml b/tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml similarity index 51% rename from tools/tests/tester/fixtures/stories/skipped/distributions.yaml rename to tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml index 8efebbb2f..27e8776dc 100644 --- a/tools/tests/tester/fixtures/stories/skipped/distributions.yaml +++ b/tools/tests/tester/fixtures/stories/skipped/distributions/included.yaml @@ -1,9 +1,9 @@ -$schema: ../../../../../../json_schemas/test_story.schema.yaml +$schema: ../../../../../../../json_schemas/test_story.schema.yaml -description: This story should be skipped because of distributions. +description: This story should be skipped because of distributions (included). distributions: - - another - - some + included: + - amazon-managed prologues: [] epilogues: [] chapters: [] \ No newline at end of file diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index d6d7e5639..1a38c4c6b 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -122,12 +122,13 @@ export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation { }) as T } - return { - ...evaluation, - chapters: flatten_chapters(evaluation.chapters), - epilogues: flatten_chapters(evaluation.epilogues), - prologues: flatten_chapters(evaluation.prologues) - } + const result = evaluation + + if (evaluation.chapters !== undefined) result.chapters = flatten_chapters(evaluation.chapters) + if (evaluation.epilogues !== undefined) result.epilogues = flatten_chapters(evaluation.epilogues) + if (evaluation.prologues !== undefined) result.prologues = flatten_chapters(evaluation.prologues) + + return result } export function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Omit & { full_path?: string } { diff --git a/tools/tests/tester/integ/TestRunner.test.ts b/tools/tests/tester/integ/TestRunner.test.ts index 0259c29ac..c1b3e2196 100644 --- a/tools/tests/tester/integ/TestRunner.test.ts +++ b/tools/tests/tester/integ/TestRunner.test.ts @@ -28,17 +28,20 @@ test('stories folder', async () => { actual_evaluations.push(rest) } - const passed = load_expected_evaluation('passed', true) - const skipped_semver = load_expected_evaluation('skipped/semver', true) - const skipped_distributions = load_expected_evaluation('skipped/distributions', true) - const not_found = load_expected_evaluation('failed/not_found', true) - const invalid_data = load_expected_evaluation('failed/invalid_data', true) - const chapter_error = load_expected_evaluation('error/chapter_error', true) - const output_error = load_expected_evaluation('error/output_error', true) - const prologue_error = load_expected_evaluation('error/prologue_error', true) - - const expected_evaluations = [passed, chapter_error, output_error, prologue_error, invalid_data, not_found, skipped_distributions, skipped_semver] - expect(actual_evaluations).toEqual(expected_evaluations) + const expected_evaluations = [ + 'passed', + 'error/chapter_error', + 'error/output_error', + 'error/prologue_error', + 'failed/invalid_data', + 'failed/not_found', + 'skipped/semver', + 'skipped/distributions/chapters', + 'skipped/distributions/excluded', + 'skipped/distributions/included' + ].map((fixture) => { return load_expected_evaluation(fixture, true) }) + + expect(actual_evaluations).toStrictEqual(expected_evaluations) }) describe('story_files', () => { From a322160b6911bab2859dc3914082480338777a29 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 15 Aug 2024 08:16:40 -0400 Subject: [PATCH 2/8] Specify warnings: at story level. (#511) Signed-off-by: dblock --- TESTING_GUIDE.md | 2 +- json_schemas/test_story.schema.yaml | 2 ++ tests/default/ingest/pipeline/neural_search.yaml | 2 ++ tools/src/tester/StoryEvaluator.ts | 16 +++++++++------- tools/src/tester/types/story.types.ts | 1 + .../fixtures/evals/error/prologue_error.yaml | 5 ----- .../fixtures/stories/error/prologue_error.yaml | 2 ++ 7 files changed, 17 insertions(+), 13 deletions(-) diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index bda054ab7..898b90be3 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -302,7 +302,7 @@ WARNING Multiple paths detected, please group similar tests together and move pa #### Suppressing Warnings -The test runner may generate warnings that can be suppressed with `warnings:`. For example, to suppress the multiple paths detected warning. +The test runner may generate warnings that can be suppressed with `warnings:` at story or chapter level. For example, to suppress the multiple paths detected warning. ```yaml - synopsis: Create an index. diff --git a/json_schemas/test_story.schema.yaml b/json_schemas/test_story.schema.yaml index b41a904ef..3a6370c43 100644 --- a/json_schemas/test_story.schema.yaml +++ b/json_schemas/test_story.schema.yaml @@ -23,6 +23,8 @@ properties: $ref: '#/definitions/Version' distributions: $ref: '#/definitions/Distributions' + warnings: + $ref: '#/definitions/Warnings' required: [chapters,description] additionalProperties: false diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 43d476e64..8168c525e 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -6,6 +6,8 @@ distributions: excluded: - amazon-managed - amazon-serverless +warnings: + multiple-paths-detected: false prologues: - path: /_cluster/settings method: PUT diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index 46371a503..fa1e46918 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -61,6 +61,7 @@ export default class StoryEvaluator { if (variables_error !== undefined) { return variables_error } + const story_outputs = new StoryOutputs() const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs) const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs, version, distribution) @@ -76,7 +77,7 @@ export default class StoryEvaluator { result: overall_result(prologues.concat(chapters).concat(epilogues).concat(prologues).map(e => e.overall)), } - const warnings = this.#chapter_warnings(story.chapters) + const warnings = this.#chapter_warnings(story) if (warnings !== undefined) { result.warnings = warnings } @@ -84,17 +85,18 @@ export default class StoryEvaluator { return result } - #chapter_warnings(chapters: Chapter[]): string[] | undefined { + #chapter_warnings(story: Story): string[] | undefined { const result = _.compact([ - this.#warning_if_mismatched_chapter_paths(chapters) + this.#warning_if_mismatched_chapter_paths(story) ]) return result.length > 0 ? result : undefined } - #warning_if_mismatched_chapter_paths(chapters: Chapter[]): string | undefined { - const paths = _.compact(_.map(chapters, (chapter) => { - const multiple_paths_detected = chapter.warnings?.['multiple-paths-detected'] ?? true - if (multiple_paths_detected) return chapter.path + #warning_if_mismatched_chapter_paths(story: Story): string | undefined { + if (story.warnings?.['multiple-paths-detected'] === false) return + const paths = _.compact(_.map(story.chapters, (chapter) => { + if (chapter.warnings?.['multiple-paths-detected'] === false) return + return chapter.path })) const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/')) const paths_counts: Record = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } }))) diff --git a/tools/src/tester/types/story.types.ts b/tools/src/tester/types/story.types.ts index e1937b112..eabb7660b 100644 --- a/tools/src/tester/types/story.types.ts +++ b/tools/src/tester/types/story.types.ts @@ -103,6 +103,7 @@ export interface Story { chapters: Chapter[]; version?: Version; distributions?: Distributions; + warnings?: Warnings; } /** * This interface was referenced by `Story`'s JSON-Schema diff --git a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml index d91beb82f..c27415bf3 100644 --- a/tools/tests/tester/fixtures/evals/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/prologue_error.yaml @@ -3,11 +3,6 @@ full_path: tools/tests/tester/fixtures/stories/error/prologue_error.yaml result: ERROR description: This story should failed due to missing info in the spec. -warnings: - - | - Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues. - /_cat/health - /_cat/indices prologues: - title: PUT /books overall: diff --git a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml index 8b11deb92..b3fd6b8bc 100644 --- a/tools/tests/tester/fixtures/stories/error/prologue_error.yaml +++ b/tools/tests/tester/fixtures/stories/error/prologue_error.yaml @@ -1,6 +1,8 @@ $schema: ../../../../../../json_schemas/test_story.schema.yaml description: This story should failed due to missing info in the spec. +warnings: + multiple-paths-detected: false prologues: - path: /books method: PUT From 3b02ba3af21935c0c17c2e7e307c766697f42e93 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 15 Aug 2024 15:08:02 -0400 Subject: [PATCH 3/8] Added detailed test coverage report. (#513) * Added detailed test coverage report. Signed-off-by: dblock --- .../pr-test-coverage-analysis.template.md | 6 +-- CHANGELOG.md | 1 + TESTING_GUIDE.md | 32 +++++++++++++ tools/src/tester/ChapterEvaluator.ts | 4 ++ tools/src/tester/ResultLogger.ts | 28 ++++++++++- tools/src/tester/StoryEvaluator.ts | 4 +- tools/src/tester/TestResults.ts | 46 ++++++++++++++----- tools/src/tester/test.ts | 2 + tools/src/tester/types/eval.types.ts | 6 +++ tools/src/tester/types/test.types.ts | 4 +- tools/tests/tester/ResultLogger.test.ts | 45 +++++++++++++++++- tools/tests/tester/TestResults.test.ts | 40 ++++++++++++---- .../fixtures/evals/error/chapter_error.yaml | 3 ++ .../fixtures/evals/error/output_error.yaml | 3 ++ .../fixtures/evals/failed/invalid_data.yaml | 15 ++++++ .../fixtures/evals/failed/not_found.yaml | 9 ++++ tools/tests/tester/fixtures/evals/passed.yaml | 30 ++++++++++++ .../tests/tester/integ/StoryEvaluator.test.ts | 4 ++ 18 files changed, 253 insertions(+), 29 deletions(-) diff --git a/.github/pr-comment-templates/pr-test-coverage-analysis.template.md b/.github/pr-comment-templates/pr-test-coverage-analysis.template.md index 27f0fe7b5..212dd595b 100644 --- a/.github/pr-comment-templates/pr-test-coverage-analysis.template.md +++ b/.github/pr-comment-templates/pr-test-coverage-analysis.template.md @@ -1,8 +1,8 @@ ## Spec Test Coverage Analysis {{with .test_coverage}} -| Total | Tested | -|-------------------|----------------------------------------------------------| -| {{.paths_count}} | {{.evaluated_paths_count}} ({{.evaluated_paths_pct}} %) | +| Total | Tested | +|------------------------------|---------------------------------------------------------------| +| {{.total_operations_count}} | {{.evaluated_operations_count}} ({{.evaluated_paths_pct}} %) | {{end}} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index b8fd3cefc..ebfbabd1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) - Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) - Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Added detailed test coverage report ([#513](https://github.com/opensearch-project/opensearch-api-specification/pull/513)) ### Changed diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 898b90be3..9aa53fca2 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -16,6 +16,9 @@ - [Warnings](#warnings) - [multiple-paths-detected](#multiple-paths-detected) - [Suppressing Warnings](#suppressing-warnings) + - [Collecting Test Coverage](#collecting-test-coverage) + - [Coverage Summary](#coverage-summary) + - [Coverage Report](#coverage-report) # Spec Testing Guide @@ -318,3 +321,32 @@ The test runner may generate warnings that can be suppressed with `warnings:` at parameters: index: movies ``` + +## Collecting Test Coverage + +### Coverage Summary + +The test tool can generate a test coverage summary using `--coverage ` with the number of evaluated verb + path combinations, a total number of paths and the % of paths tested. + +```json +{ + "evaluated_operations_count": 214, + "operations_count": 550, + "evaluated_paths_pct": 38.91 +} +``` + +The report is then used by the [test-spec.yml](.github/workflows/test-spec.yml) workflow, uploaded with every run, combined across various versions of OpenSearch, and reported as a comment on each pull request. + +### Coverage Report + +The test tool can display detailed and hierarchal test coverage with `--coverage-report`. This is useful to identify untested paths. For example, the [put_alias.yaml](tests/default/indices/aliases/put_alias.yaml) test exercises `PUT /_alias/{name}`, but not the other verbs. The report produces the following output with the missing ones. + +``` +/_alias (4) + GET /_alias + /{name} (3) + GET /_alias/{name} + POST /_alias/{name} + HEAD /_alias/{name} +``` \ No newline at end of file diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index dee177ddc..6feb7d0a0 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -84,6 +84,10 @@ export default class ChapterEvaluator { var result: ChapterEvaluation = { title: chapter.synopsis, + operation: { + method: chapter.method, + path: chapter.path + }, path: `${chapter.method} ${chapter.path}`, overall: { result: overall_result(evaluations) }, request: { parameters: params, request }, diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts index d2ef339c2..0d6d2c929 100644 --- a/tools/src/tester/ResultLogger.ts +++ b/tools/src/tester/ResultLogger.ts @@ -7,10 +7,11 @@ * compatible open source license. */ -import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' +import { type ChapterEvaluation, type Evaluation, Operation, Result, type StoryEvaluation } from './types/eval.types' import { overall_result } from './helpers' import * as ansi from './Ansi' import TestResults from './TestResults' +import _ from 'lodash' export interface ResultLogger { log: (evaluation: StoryEvaluation) => void @@ -43,7 +44,30 @@ export class ConsoleResultLogger implements ResultLogger { log_coverage(results: TestResults): void { console.log() - console.log(`Tested ${results.evaluated_paths_count()}/${results.spec_paths_count()} paths.`) + console.log(`Tested ${results.evaluated_operations().length}/${results.operations().length} paths.`) + } + + log_coverage_report(results: TestResults): void { + console.log() + console.log(`${results.unevaluated_operations().length} paths remaining.`) + const groups = _.groupBy(results.unevaluated_operations(), (operation) => operation.path.split('/')[1]) + Object.entries(groups).forEach(([root, operations]) => { + this.#log_coverage_group(root, operations) + }); + } + + #log_coverage_group(key: string, operations: Operation[], index: number = 2): void { + if (operations.length == 0 || key == undefined) return + console.log(`${' '.repeat(index)}/${key} (${operations.length})`) + const current_level_operations = operations.filter((operation) => operation.path.split('/').length == index) + current_level_operations.forEach((operation) => { + console.log(`${' '.repeat(index + 2)}${operation.method} ${operation.path}`) + }) + const next_level_operations = operations.filter((operation) => operation.path.split('/').length > index) + const subgroups = _.groupBy(next_level_operations, (operation) => operation.path.split('/')[index]) + Object.entries(subgroups).forEach(([root, operations]) => { + this.#log_coverage_group(root, operations, index + 1) + }); } #log_story ({ result, full_path, display_path, message, warnings }: StoryEvaluation): void { diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index fa1e46918..e464f3f60 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -99,8 +99,8 @@ export default class StoryEvaluator { return chapter.path })) const normalized_paths = _.map(paths, (path) => path.replaceAll(/\/\{[^}]+}/g, '').replaceAll('//', '/')) - const paths_counts: Record = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } }))) - if (paths_counts.length > 1) { + const operations_counts: Record = Object.assign((_.values(_.groupBy(normalized_paths)).map(p => { return { [p[0]] : p.length } }))) + if (operations_counts.length > 1) { return `Multiple paths detected, please group similar tests together and move paths not being tested to prologues or epilogues.\n ${_.join(_.uniq(paths), "\n ")}\n` } } diff --git a/tools/src/tester/TestResults.ts b/tools/src/tester/TestResults.ts index c4ae26b78..792699e20 100644 --- a/tools/src/tester/TestResults.ts +++ b/tools/src/tester/TestResults.ts @@ -9,35 +9,59 @@ import _ from "lodash"; import MergedOpenApiSpec from "./MergedOpenApiSpec"; -import { StoryEvaluations } from "./types/eval.types"; +import { Operation, StoryEvaluations } from "./types/eval.types"; import { SpecTestCoverage } from "./types/test.types"; import { write_json } from "../helpers"; export default class TestResults { protected _spec: MergedOpenApiSpec protected _evaluations: StoryEvaluations + protected _evaluated_operations?: Operation[] + protected _unevaluated_operations?: Operation[] + protected _operations?: Operation[] constructor(spec: MergedOpenApiSpec, evaluations: StoryEvaluations) { this._spec = spec this._evaluations = evaluations } - evaluated_paths_count(): number { - return _.uniq(_.compact(_.flatten(_.map(this._evaluations.evaluations, (evaluation) => - _.map(evaluation.chapters, (chapter) => chapter.path) - )))).length + evaluated_operations(): Operation[] { + if (this._evaluated_operations !== undefined) return this._evaluated_operations + this._evaluated_operations = _.uniq(_.compact(_.flatMap(this._evaluations.evaluations, (evaluation) => + _.map(evaluation.chapters, (chapter) => chapter.operation) + ))) + return this._evaluated_operations } - spec_paths_count(): number { - return Object.values(this._spec.paths()).reduce((acc, methods) => acc + methods.length, 0); + unevaluated_operations(): Operation[] { + if (this._unevaluated_operations !== undefined) return this._unevaluated_operations + this._unevaluated_operations = this.operations().filter((operation) => + !_.find(this.evaluated_operations(), + (op) => + operation.method == op.method && + operation.path == op.path + ) + ) + return this._unevaluated_operations + } + + operations(): Operation[] { + if (this._operations !== undefined) return this._operations + this._operations = _.uniq(Object.entries(this._spec.paths()).flatMap(([path, path_item]) => { + return Object.values(path_item).map((method) => { + return { method: method.toUpperCase(), path } + }) + })) + + return this._operations } test_coverage(): SpecTestCoverage { return { - evaluated_paths_count: this.evaluated_paths_count(), - paths_count: this.spec_paths_count(), - evaluated_paths_pct: this.spec_paths_count() > 0 ? Math.round( - this.evaluated_paths_count() / this.spec_paths_count() * 100 * 100 + evaluated_operations_count: this.evaluated_operations().length, + total_operations_count: this.operations().length, + evaluated_paths_pct: this.operations().length > 0 ? Math.round( + this.evaluated_operations().length / this.operations().length * 100 * 100 ) / 100 : 0, } } diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index da025a34b..e781e343c 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -60,6 +60,7 @@ const command = new Command() .addOption(AWS_REGION_OPTION) .addOption(AWS_SERVICE_OPTION) .addOption(new Option('--coverage ', 'path to write test coverage results to')) + .addOption(new Option('--coverage-report', 'display a detailed test coverage report')) .allowExcessArguments(false) .parse() @@ -82,6 +83,7 @@ runner.run(opts.testsPath, spec.api_version(), opts.opensearchDistribution, opts const test_results = new TestResults(spec, results) result_logger.log_coverage(test_results) + if (opts.coverageReport) result_logger.log_coverage_report(test_results) if (opts.coverage !== undefined) { console.log(`Writing ${opts.coverage} ...`) test_results.write_coverage(opts.coverage) diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index a95104c50..eacb2c62a 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -17,6 +17,11 @@ export interface StoryFile { story: Story } +export interface Operation { + method: string + path: string +} + export interface StoryEvaluation { result: Result display_path: string @@ -36,6 +41,7 @@ export interface StoryEvaluations { export interface ChapterEvaluation { title: string, overall: Evaluation, + operation?: Operation, path?: string, request?: { parameters?: Record diff --git a/tools/src/tester/types/test.types.ts b/tools/src/tester/types/test.types.ts index c0e14717a..174f207c0 100644 --- a/tools/src/tester/types/test.types.ts +++ b/tools/src/tester/types/test.types.ts @@ -8,7 +8,7 @@ */ export interface SpecTestCoverage { - paths_count: number - evaluated_paths_count: number, + total_operations_count: number + evaluated_operations_count: number, evaluated_paths_pct: number } diff --git a/tools/tests/tester/ResultLogger.test.ts b/tools/tests/tester/ResultLogger.test.ts index f21e45af1..512619cd9 100644 --- a/tools/tests/tester/ResultLogger.test.ts +++ b/tools/tests/tester/ResultLogger.test.ts @@ -66,7 +66,11 @@ describe('ConsoleResultLogger', () => { { title: 'title', overall: { result: Result.PASSED }, - path: 'path' + path: 'path', + operation: { + method: 'GET', + path: '/_nodes/{id}' + } } ] }] }) @@ -79,6 +83,45 @@ describe('ConsoleResultLogger', () => { ]) }) + test('log_coverage_report', () => { + const spec = new MergedOpenApiSpec('tools/tests/tester/fixtures/specs/complete') + const test_results = new TestResults(spec, { evaluations: [{ + result: Result.PASSED, + display_path: 'path', + full_path: 'path', + description: 'description', + chapters: [ + { + title: 'title', + overall: { result: Result.PASSED }, + path: 'GET /_nodes/{id}', + operation: { + method: 'GET', + path: '/_nodes/{id}' + } + } + ] + }] }) + + logger.log_coverage_report(test_results) + + expect(log.mock.calls).not.toContain(["GET /_nodes/{id}"]) + expect(log.mock.calls).toEqual([ + [], + ["5 paths remaining."], + [" /_nodes (1)"], + [" /{id} (1)"], + [" POST /_nodes/{id}"], + [" /cluster_manager (2)"], + [" GET /cluster_manager"], + [" POST /cluster_manager"], + [" /index (1)"], + [" GET /index"], + [" /nodes (1)"], + [" GET /nodes"] + ]) + }) + test('with retries', () => { logger.log({ result: Result.PASSED, diff --git a/tools/tests/tester/TestResults.test.ts b/tools/tests/tester/TestResults.test.ts index ac06a6350..d4abe8472 100644 --- a/tools/tests/tester/TestResults.test.ts +++ b/tools/tests/tester/TestResults.test.ts @@ -17,16 +17,20 @@ describe('TestResults', () => { const evaluations = [{ result: Result.PASSED, - display_path: 'path', + display_path: 'PUT /{index}', full_path: 'full_path', description: 'description', message: 'message', chapters: [{ title: 'title', + operation: { + method: 'PUT', + path: '/{index}' + }, overall: { result: Result.PASSED }, - path: 'path' + path: 'PUT /{index}' }], epilogues: [], prologues: [] @@ -34,21 +38,41 @@ describe('TestResults', () => { const test_results = new TestResults(spec, { evaluations }) - test('evaluated_paths_count', () => { - expect(test_results.evaluated_paths_count()).toEqual(1) + test('unevaluated_operations', () => { + expect(test_results.unevaluated_operations()).toEqual([ + { method: "GET", path: "/_nodes/{id}" }, + { method: "POST", path: "/_nodes/{id}" }, + { method: "GET", path: "/cluster_manager" }, + { method: "POST", path: "/cluster_manager" }, + { method: "GET", path: "/index" }, + { method: "GET", path: "/nodes" } + ]) + }) + + test('evaluated_operations', () => { + expect(test_results.evaluated_operations()).toStrictEqual([ + { method: 'PUT', path: '/{index}' } + ]) }) - test('spec_paths_count', () => { - expect(test_results.spec_paths_count()).toEqual(6) + test('operations', () => { + expect(test_results.operations()).toEqual([ + { method: "GET", path: "/_nodes/{id}" }, + { method: "POST", path: "/_nodes/{id}" }, + { method: "GET", path: "/cluster_manager" }, + { method: "POST", path: "/cluster_manager" }, + { method: "GET", path: "/index" }, + { method: "GET", path: "/nodes" } + ]) }) test('write_coverage', () => { const filename = 'coverage.json' test_results.write_coverage(filename) expect(JSON.parse(fs.readFileSync(filename, 'utf8'))).toEqual({ - evaluated_paths_count: 1, + evaluated_operations_count: 1, evaluated_paths_pct: 16.67, - paths_count: 6 + total_operations_count: 6 }) fs.unlinkSync(filename) }) diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index fab1f0cc4..331667e99 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -24,6 +24,9 @@ chapters: overall: result: ERROR path: DELETE /{index} + operation: + method: DELETE + path: /{index} request: parameters: {} request: diff --git a/tools/tests/tester/fixtures/evals/error/output_error.yaml b/tools/tests/tester/fixtures/evals/error/output_error.yaml index 27ac940d3..5a7e03bb3 100644 --- a/tools/tests/tester/fixtures/evals/error/output_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/output_error.yaml @@ -11,6 +11,9 @@ chapters: overall: result: ERROR path: GET /_cat/health + operation: + method: GET + path: /_cat/health retries: 3 request: parameters: diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml index 3f59fe0e8..14716a69b 100644 --- a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml @@ -9,6 +9,9 @@ chapters: overall: result: FAILED path: PUT /{index} + operation: + method: PUT + path: /{index} request: parameters: index: @@ -29,6 +32,9 @@ chapters: overall: result: FAILED path: PUT /{index} + operation: + method: PUT + path: /{index} request: parameters: index: @@ -49,6 +55,9 @@ chapters: overall: result: FAILED path: GET /_cat/indices/{index} + operation: + method: GET + path: /_cat/indices/{index} request: parameters: format: @@ -71,6 +80,9 @@ chapters: overall: result: FAILED path: DELETE /{index} + operation: + method: DELETE + path: /{index} request: parameters: index: @@ -92,6 +104,9 @@ chapters: overall: result: ERROR path: PUT /{index} + operation: + method: PUT + path: /{index} request: parameters: index: diff --git a/tools/tests/tester/fixtures/evals/failed/not_found.yaml b/tools/tests/tester/fixtures/evals/failed/not_found.yaml index ecbd88992..9a25ad3ee 100644 --- a/tools/tests/tester/fixtures/evals/failed/not_found.yaml +++ b/tools/tests/tester/fixtures/evals/failed/not_found.yaml @@ -18,6 +18,9 @@ chapters: overall: result: FAILED path: PUT /{index} + operation: + method: PUT + path: /{index} request: parameters: index: @@ -40,6 +43,9 @@ chapters: overall: result: FAILED path: HEAD /{index} + operation: + method: HEAD + path: /{index} request: parameters: index: @@ -60,6 +66,9 @@ chapters: overall: result: FAILED path: DELETE /{index} + operation: + method: DELETE + path: /{index} request: parameters: index: diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index abe502379..e20d6fc4a 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -9,6 +9,9 @@ chapters: overall: result: PASSED path: PUT /{index} + operation: + method: PUT + path: /{index} request: parameters: index: @@ -28,6 +31,9 @@ chapters: overall: result: PASSED path: GET /_cat + operation: + method: GET + path: /_cat request: parameters: {} request: @@ -45,6 +51,9 @@ chapters: overall: result: PASSED path: GET /_cat + operation: + method: GET + path: /_cat request: parameters: {} request: @@ -62,6 +71,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -81,6 +93,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -100,6 +115,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -119,6 +137,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -138,6 +159,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -157,6 +181,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: @@ -180,6 +207,9 @@ chapters: overall: result: PASSED path: GET /_cat/health + operation: + method: GET + path: /_cat/health request: parameters: format: diff --git a/tools/tests/tester/integ/StoryEvaluator.test.ts b/tools/tests/tester/integ/StoryEvaluator.test.ts index ac5fc60ef..f3c6bc15e 100644 --- a/tools/tests/tester/integ/StoryEvaluator.test.ts +++ b/tools/tests/tester/integ/StoryEvaluator.test.ts @@ -70,6 +70,10 @@ test('with an unexpected error deserializing data', async () => { expect(actual.chapters && actual.chapters[0]).toEqual({ title: "This PUT /{index} chapter should pass.", path: 'PUT /{index}', + operation: { + method: 'PUT', + path: '/{index}' + }, overall: { result: Result.ERROR }, From 52ec3755965bd6882e4566722e130f180b9aa61f Mon Sep 17 00:00:00 2001 From: Uriel Dan Nudelman Date: Mon, 19 Aug 2024 09:00:07 -0400 Subject: [PATCH 4/8] Fixes NeuralQuery Schema (#512) * Fixes NeuralQuery Schema Signed-off-by: uri.nudelman * Remove unsupported validation Signed-off-by: uri.nudelman * Adds encoding base64 Signed-off-by: uri.nudelman * Add CHANGELOG Signed-off-by: uri.nudelman * Adding Test Case Signed-off-by: uri.nudelman * Adding Tests and Error Responses to Schema Signed-off-by: uri.nudelman --------- Signed-off-by: uri.nudelman Co-authored-by: uri.nudelman --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 28 ++++++++++++++++++- spec/schemas/_common.query_dsl.yaml | 4 +-- spec/schemas/query._common.yaml | 8 +++--- .../ingest/pipeline/neural_search.yaml | 19 ++++++++++++- 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebfbabd1c..679e570e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed accuracy of the index stats schemas ([#491](https://github.com/opensearch-project/opensearch-api-specification/pull/491)) - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439)) - Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) +- Fixed query DSL `neural` field `query_image` set `contentEncoding` and `model_id` as optional ([#512](https://github.com/opensearch-project/opensearch-api-specification/pull/512)) ### Security diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index bc90411bb..dc0089b29 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -696,6 +696,10 @@ paths: responses: '200': $ref: '#/components/responses/search@200' + '400': + $ref: '#/components/responses/search@400' + '404': + $ref: '#/components/responses/search@404' post: operationId: search.1 x-operation-group: search @@ -755,6 +759,10 @@ paths: responses: '200': $ref: '#/components/responses/search@200' + '400': + $ref: '#/components/responses/search@400' + '404': + $ref: '#/components/responses/search@404' /_search/point_in_time: delete: operationId: delete_pit.0 @@ -1724,6 +1732,10 @@ paths: responses: '200': $ref: '#/components/responses/search@200' + '400': + $ref: '#/components/responses/search@400' + '404': + $ref: '#/components/responses/search@404' post: operationId: search.3 x-operation-group: search @@ -1784,6 +1796,10 @@ paths: responses: '200': $ref: '#/components/responses/search@200' + '400': + $ref: '#/components/responses/search@400' + '404': + $ref: '#/components/responses/search@404' /{index}/_search/point_in_time: post: operationId: create_pit.0 @@ -3044,6 +3060,16 @@ components: application/json: schema: $ref: '../schemas/_core.search.yaml#/components/schemas/ResponseBody' + search@400: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/ErrorResponse' + search@404: + content: + application/json: + schema: + $ref: '../schemas/query._common.yaml#/components/schemas/ErrorResponse' search_shards@200: content: application/json: @@ -6259,4 +6285,4 @@ components: description: The throttle for this request in sub-requests per second. schema: type: number - style: form + style: form \ No newline at end of file diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index a4bb57b06..28991fc6b 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -1237,7 +1237,7 @@ components: type: string query_image: type: string - format: binary + contentEncoding: base64 model_id: type: string k: @@ -1248,8 +1248,6 @@ components: type: number filter: $ref: '#/components/schemas/QueryContainer' - required: - - model_id ParentIdQuery: allOf: - $ref: '#/components/schemas/QueryBase' diff --git a/spec/schemas/query._common.yaml b/spec/schemas/query._common.yaml index 080c3afa3..40277b8b0 100644 --- a/spec/schemas/query._common.yaml +++ b/spec/schemas/query._common.yaml @@ -79,10 +79,10 @@ components: $ref: '#/components/schemas/RootCause' type: type: string - example: status_exception + # example: status_exception reason: type: string - example: DataSource not found + # example: DataSource not found required: - reason - root_cause @@ -93,10 +93,10 @@ components: properties: type: type: string - example: status_exception + # example: status_exception reason: type: string - example: DataSource not found + # example: DataSource not found required: - reason - type diff --git a/tests/default/ingest/pipeline/neural_search.yaml b/tests/default/ingest/pipeline/neural_search.yaml index 8168c525e..e99d10f23 100644 --- a/tests/default/ingest/pipeline/neural_search.yaml +++ b/tests/default/ingest/pipeline/neural_search.yaml @@ -165,7 +165,7 @@ chapters: - {director: Nicolas Winding Refn, title: Drive, year: 1960} response: status: 200 - - synopsis: Search. + - synopsis: Search by Text. path: /{index}/_search method: POST parameters: @@ -192,6 +192,23 @@ chapters: title: Moneyball script: source: _score * 1.7 + - synopsis: Search by Image - Invalid Model. + path: /{index}/_search + method: POST + parameters: + index: movies + request: + payload: + _source: + excludes: [passage_embedding] + query: + neural: + passage_embedding: + query_image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII= + model_id: invalid # This makes the test fail with a 404 but will still validate the query_image. + k: 100 + response: + status: 404 - synopsis: Undeploy a model. path: /_plugins/_ml/models/{model_id}/_undeploy method: POST From f9e2701d9ea00216a2ec8c474129391b04328c35 Mon Sep 17 00:00:00 2001 From: kolchfa-aws <105444904+kolchfa-aws@users.noreply.github.com> Date: Mon, 19 Aug 2024 12:57:56 -0400 Subject: [PATCH 5/8] Refactor the common query DSL spec to extract IgnoreUnmapped (#515) Signed-off-by: Fanit Kolchina --- spec/schemas/_common.query_dsl.yaml | 35 +++++++++++++---------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/spec/schemas/_common.query_dsl.yaml b/spec/schemas/_common.query_dsl.yaml index 28991fc6b..ba5afcc3b 100644 --- a/spec/schemas/_common.query_dsl.yaml +++ b/spec/schemas/_common.query_dsl.yaml @@ -593,6 +593,12 @@ components: - type: boolean required: - value + IgnoreUnmapped: + type: boolean + default: false + description: |- + Set to `true` to ignore an unmapped field and not match any documents for this query. + Set to `false` to throw an exception if the field is not mapped. GeoBoundingBoxQuery: allOf: - $ref: '#/components/schemas/QueryBase' @@ -603,10 +609,7 @@ components: validation_method: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: - description: |- - Set to `true` to ignore an unmapped field and not match any documents for this query. - Set to `false` to throw an exception if the field is not mapped. - type: boolean + $ref: '#/components/schemas/IgnoreUnmapped' GeoExecution: type: string enum: @@ -630,11 +633,7 @@ components: validation_method: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: - description: |- - Set to `true` to ignore an unmapped field and not match any documents for this query. - Set to `false` to throw an exception if the field is not mapped. - type: boolean - default: false + $ref: '#/components/schemas/IgnoreUnmapped' field: type: object required: @@ -648,25 +647,22 @@ components: validation_method: $ref: '#/components/schemas/GeoValidationMethod' ignore_unmapped: - type: boolean + $ref: '#/components/schemas/IgnoreUnmapped' GeoShapeQuery: allOf: - $ref: '#/components/schemas/QueryBase' - type: object properties: ignore_unmapped: - description: |- - Set to `true` to ignore an unmapped field and not match any documents for this query. - Set to `false` to throw an exception if the field is not mapped. - type: boolean + $ref: '#/components/schemas/IgnoreUnmapped' HasChildQuery: allOf: - $ref: '#/components/schemas/QueryBase' - type: object properties: ignore_unmapped: + $ref: '#/components/schemas/IgnoreUnmapped' description: Indicates whether to ignore an unmapped `type` and not return any documents instead of an error. - type: boolean inner_hits: $ref: '_core.search.yaml#/components/schemas/InnerHits' max_children: @@ -714,10 +710,10 @@ components: - type: object properties: ignore_unmapped: + $ref: '#/components/schemas/IgnoreUnmapped' description: |- Indicates whether to ignore an unmapped `parent_type` and not return any documents instead of an error. You can use this parameter to query multiple indices that may not contain the `parent_type`. - type: boolean inner_hits: $ref: '_core.search.yaml#/components/schemas/InnerHits' parent_type: @@ -1211,8 +1207,8 @@ components: - type: object properties: ignore_unmapped: + $ref: '#/components/schemas/IgnoreUnmapped' description: Indicates whether to ignore an unmapped path and not return any documents instead of an error. - type: boolean inner_hits: $ref: '_core.search.yaml#/components/schemas/InnerHits' path: @@ -1256,8 +1252,8 @@ components: id: $ref: '_common.yaml#/components/schemas/Id' ignore_unmapped: + $ref: '#/components/schemas/IgnoreUnmapped' description: Indicates whether to ignore an unmapped `type` and not return any documents instead of an error. - type: boolean type: $ref: '_common.yaml#/components/schemas/RelationName' PercolateQuery: @@ -1611,8 +1607,7 @@ components: - type: object properties: ignore_unmapped: - description: When set to `true` the query ignores an unmapped field and will not match any documents. - type: boolean + $ref: '#/components/schemas/IgnoreUnmapped' SimpleQueryStringQuery: allOf: - $ref: '#/components/schemas/QueryBase' From ceefc370e81d38e56820a6f071695098573dceab Mon Sep 17 00:00:00 2001 From: David Zane <38449481+dzane17@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:39:12 -0700 Subject: [PATCH 6/8] Added nodes/stats indices search test (#509) Signed-off-by: David Zane Signed-off-by: David Zane <38449481+dzane17@users.noreply.github.com> --- tests/default/nodes/stats.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/default/nodes/stats.yaml b/tests/default/nodes/stats.yaml index a52225be7..7674f463f 100644 --- a/tests/default/nodes/stats.yaml +++ b/tests/default/nodes/stats.yaml @@ -166,5 +166,15 @@ chapters: - indices index_metric: - request_cache + response: + status: 200 + - synopsis: Get statistics for search. + path: /_nodes/stats/{metric}/{index_metric} + method: GET + parameters: + metric: + - indices + index_metric: + - search response: status: 200 \ No newline at end of file From 772b1c89e20afa52ccc3fa3449ae9152aefa4394 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 21 Aug 2024 14:20:32 -0400 Subject: [PATCH 7/8] Complete variations of /{index}/alias/{name} tests, add tests for HEAD, bulk, analyzers. (#519) * Added HEAD / tests. * Detailed where to start adding tests by using the test coverage report. * Complete alias tests. * Added tests for PUT /_bulk and /{index}/_bulk. * Added tests for /_analyze and /{index}/_analyze. Signed-off-by: dblock --- CHANGELOG.md | 1 + TESTING_GUIDE.md | 8 +- spec/namespaces/indices.yaml | 18 ++++ tests/default/_core/alias.yaml | 84 ++++++++++++++++++ tests/default/_core/aliases.yaml | 59 +++++++++++++ tests/default/_core/analyze.yaml | 92 ++++++++++++++++++++ tests/default/_core/bulk.yaml | 17 ++++ tests/default/_core/info.yaml | 5 ++ tests/default/indices/alias/alias.yaml | 20 ++++- tests/default/indices/aliases/aliases.yaml | 21 +---- tests/default/indices/aliases/put_alias.yaml | 79 ----------------- tests/default/indices/analyze.yaml | 51 +++++++++++ tests/default/indices/bulk.yaml | 57 ++++++++++++ 13 files changed, 409 insertions(+), 103 deletions(-) create mode 100644 tests/default/_core/alias.yaml create mode 100644 tests/default/_core/aliases.yaml create mode 100644 tests/default/_core/analyze.yaml delete mode 100644 tests/default/indices/aliases/put_alias.yaml create mode 100644 tests/default/indices/analyze.yaml create mode 100644 tests/default/indices/bulk.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 679e570e6..c5a91329c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) - Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483)) - Added detailed test coverage report ([#513](https://github.com/opensearch-project/opensearch-api-specification/pull/513)) +- Added 404 responses to `/_alias/{name}` and `/{index}/_alias/{name}` ([#519](https://github.com/opensearch-project/opensearch-api-specification/pull/519)) ### Changed diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 9aa53fca2..852d3844b 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -40,6 +40,7 @@ docker compose up -d Run the tests (use `--opensearch-insecure` for a local cluster running in Docker that does not have a valid SSL certificate): ```bash +export OPENSEARCH_PASSWORD=<> npm run test:spec -- --opensearch-insecure ``` @@ -53,6 +54,11 @@ Verbose output: npm run test:spec -- --opensearch-insecure --verbose ``` +Want to help with some missing tests? Choose from the remaining paths in the test coverage report: +```bash +npm run test:spec -- --opensearch-insecure --coverage-report +``` + ### Running Spec Tests with Amazon OpenSearch Use an Amazon OpenSearch service instance. @@ -340,7 +346,7 @@ The report is then used by the [test-spec.yml](.github/workflows/test-spec.yml) ### Coverage Report -The test tool can display detailed and hierarchal test coverage with `--coverage-report`. This is useful to identify untested paths. For example, the [put_alias.yaml](tests/default/indices/aliases/put_alias.yaml) test exercises `PUT /_alias/{name}`, but not the other verbs. The report produces the following output with the missing ones. +The test tool can display detailed and hierarchal test coverage with `--coverage-report`. This is useful to identify untested paths. The report produces the following output with the missing ones. ``` /_alias (4) diff --git a/spec/namespaces/indices.yaml b/spec/namespaces/indices.yaml index d5e13e558..3db14ce5a 100644 --- a/spec/namespaces/indices.yaml +++ b/spec/namespaces/indices.yaml @@ -20,6 +20,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.get_alias@200' + '404': + $ref: '#/components/responses/indices.get_alias@404' put: operationId: indices.put_alias.0 x-operation-group: indices.put_alias @@ -53,6 +55,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.get_alias@200' + '404': + $ref: '#/components/responses/indices.get_alias@404' head: operationId: indices.exists_alias.0 x-operation-group: indices.exists_alias @@ -69,6 +73,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.exists_alias@200' + '404': + $ref: '#/components/responses/indices.exists_alias@404' post: operationId: indices.put_alias.1 x-operation-group: indices.put_alias @@ -1055,6 +1061,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.get_alias@200' + '404': + $ref: '#/components/responses/indices.get_alias@404' put: operationId: indices.put_alias.5 x-operation-group: indices.put_alias @@ -1090,6 +1098,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.get_alias@200' + '404': + $ref: '#/components/responses/indices.get_alias@404' head: operationId: indices.exists_alias.1 x-operation-group: indices.exists_alias @@ -1107,6 +1117,8 @@ paths: responses: '200': $ref: '#/components/responses/indices.exists_alias@200' + '404': + $ref: '#/components/responses/indices.exists_alias@404' post: operationId: indices.put_alias.6 x-operation-group: indices.put_alias @@ -2417,6 +2429,9 @@ components: indices.exists_alias@200: content: application/json: {} + indices.exists_alias@404: + content: + application/json: {} indices.exists_index_template@200: content: application/json: {} @@ -2455,6 +2470,9 @@ components: type: object additionalProperties: $ref: '../schemas/indices.get_alias.yaml#/components/schemas/IndexAliases' + indices.get_alias@404: + content: + application/json: {} indices.get_data_stream@200: content: application/json: diff --git a/tests/default/_core/alias.yaml b/tests/default/_core/alias.yaml new file mode 100644 index 000000000..2cb9e7840 --- /dev/null +++ b/tests/default/_core/alias.yaml @@ -0,0 +1,84 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test /_alias variants. +epilogues: + - path: /movies/_alias/film* + method: DELETE + status: [200, 404] + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /{index} + method: PUT + parameters: + index: movies +chapters: + - synopsis: Create an alias (index in body). + path: /_alias/{name} + method: PUT + parameters: + name: film1 + request: + payload: + index: movies + response: + status: 200 + payload: + acknowledged: true + - synopsis: Create an alias (index in body). + path: /_alias/{name} + method: POST + parameters: + name: film2 + request: + payload: + index: movies + response: + status: 200 + payload: + acknowledged: true + - synopsis: Create an alias (index & alias in body). + path: /_alias + method: PUT + request: + payload: + index: movies + alias: film3 + response: + status: 200 + payload: + acknowledged: true + - synopsis: Get all aliases. + path: /_alias + method: GET + response: + status: 200 + - synopsis: Get an alias by name. + path: /_alias/{name} + parameters: + name: film1 + method: GET + response: + status: 200 + - synopsis: Get an alias by name. + path: /_alias/{name} + parameters: + name: invalid + method: GET + response: + status: 404 + - synopsis: Check that an alias exists. + path: /_alias/{name} + parameters: + name: film1 + method: HEAD + response: + status: 200 + - synopsis: Check that an alias exists. + path: /_alias/{name} + parameters: + name: invalid + method: HEAD + response: + status: 404 diff --git a/tests/default/_core/aliases.yaml b/tests/default/_core/aliases.yaml new file mode 100644 index 000000000..9f1cee73f --- /dev/null +++ b/tests/default/_core/aliases.yaml @@ -0,0 +1,59 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test aliases endpoints. +epilogues: + - path: /games/_alias/* + method: DELETE + status: [200, 404] + - path: /games + method: DELETE + status: [200, 404] +prologues: + - path: games + method: PUT + - path: /games/_aliases/jeux + method: PUT +chapters: + - synopsis: Multiple alias operations. + path: /_aliases + method: POST + request: + payload: + actions: + - remove: + index: games + alias: jeux + - add: + index: games + alias: plays1 + - add: + index: games + alias: plays2 + response: + status: 200 + payload: + acknowledged: true + - synopsis: Create an alias. + path: /_aliases/{name} + method: POST + parameters: + name: plays3 + request: + payload: + index: games + response: + status: 200 + payload: + acknowledged: true + - synopsis: Update an alias. + path: /_aliases/{name} + method: PUT + parameters: + name: plays4 + request: + payload: + index: games + response: + status: 200 + payload: + acknowledged: true diff --git a/tests/default/_core/analyze.yaml b/tests/default/_core/analyze.yaml new file mode 100644 index 000000000..659ec9a96 --- /dev/null +++ b/tests/default/_core/analyze.yaml @@ -0,0 +1,92 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test /_analyze. +epilogues: [] +prologues: [] +chapters: + - synopsis: Analyze text. + path: /_analyze + method: POST + request: + payload: + analyzer: standard + text: Moneyball, directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: moneyball + start_offset: 0 + end_offset: 9 + position: 0 + - synopsis: Analyze text as a multi-value field. + path: /_analyze + method: GET + request: + payload: + analyzer: standard + text: + - Drive, directed by Nicolas Winding Refn + - Moneyball, directed by Bennett Miller + response: + status: 200 + - synopsis: Apply a filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - uppercase + text: Moneyball + response: + status: 200 + payload: + tokens: + - token: MONEYBALL + type: word + start_offset: 0 + end_offset: 9 + position: 0 + - synopsis: Apply a character filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: keyword + filter: + - lowercase + char_filter: + - html_strip + text: Moneyball + response: + status: 200 + payload: + tokens: + - token: moneyball + type: word + start_offset: 3 + end_offset: 16 + position: 0 + - synopsis: Combine a lowercase translation with a stop filter. + path: /_analyze + method: GET + request: + payload: + tokenizer: whitespace + filter: + - lowercase + - type: stop + stopwords: + - in + - to + text: Moneyball directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: moneyball + type: word + start_offset: 0 + end_offset: 9 + position: 0 diff --git a/tests/default/_core/bulk.yaml b/tests/default/_core/bulk.yaml index 3ed172528..37104b2b7 100644 --- a/tests/default/_core/bulk.yaml +++ b/tests/default/_core/bulk.yaml @@ -14,6 +14,23 @@ chapters: payload: - {create: {_index: movies}} - {director: Bennett Miller, title: Moneyball, year: 2011} + - synopsis: Delete document in an index. + path: /_bulk + method: PUT + request: + content_type: application/x-ndjson + payload: + - {delete: {_index: movies, _id: invalid}} + response: + status: 200 + payload: + errors: false + items: + - delete: + _index: movies + _id: invalid + result: not_found + status: 404 - synopsis: Bulk document CRUD. path: /_bulk method: POST diff --git a/tests/default/_core/info.yaml b/tests/default/_core/info.yaml index f8a360360..813ef6ab6 100644 --- a/tests/default/_core/info.yaml +++ b/tests/default/_core/info.yaml @@ -6,6 +6,11 @@ distributions: excluded: - amazon-serverless chapters: + - synopsis: Head server info. + path: / + method: HEAD + response: + status: 200 - synopsis: Get server info. path: / method: GET diff --git a/tests/default/indices/alias/alias.yaml b/tests/default/indices/alias/alias.yaml index 2fbba5383..4000dbb13 100644 --- a/tests/default/indices/alias/alias.yaml +++ b/tests/default/indices/alias/alias.yaml @@ -36,13 +36,27 @@ chapters: parameters: index: games name: jeux - - synopsis: Create an alias by Create or Update alias endpoint. + - synopsis: Create an alias. path: /{index}/_alias/{name} method: PUT parameters: index: games name: jeux - - synopsis: Create an alias with custom settings by Create or Update alias endpoint. + - synopsis: Check if alias exists. + path: /{index}/_alias/{name} + method: HEAD + parameters: + index: games + name: jeux + - synopsis: Check if alias exists. + path: /{index}/_alias/{name} + method: HEAD + parameters: + index: games + name: invalid + response: + status: 404 + - synopsis: Create an alias with custom settings. path: /{index}/_alias/{name} method: PUT parameters: @@ -56,7 +70,7 @@ chapters: is_write_index: true filter: match_all: {} - - synopsis: Create an alias with is_hidden by Create or Update alias endpoint. + - synopsis: Create an alias with is_hidden. version: '>= 2.16' path: /{index}/_alias/{name} method: PUT diff --git a/tests/default/indices/aliases/aliases.yaml b/tests/default/indices/aliases/aliases.yaml index 805510ae4..75a1918c1 100644 --- a/tests/default/indices/aliases/aliases.yaml +++ b/tests/default/indices/aliases/aliases.yaml @@ -15,28 +15,9 @@ chapters: parameters: index: games name: jeux - - synopsis: Multiple alias operations. - path: /_aliases - method: POST - request: - payload: - actions: - - remove: - index: games - alias: jeux - - add: - index: games - alias: plays1 - - add: - index: games - alias: plays2 - response: - status: 200 - payload: - acknowledged: true - synopsis: Delete an alias. path: /{index}/_aliases/{name} method: DELETE parameters: index: games - name: plays1 + name: jeux diff --git a/tests/default/indices/aliases/put_alias.yaml b/tests/default/indices/aliases/put_alias.yaml deleted file mode 100644 index d7eaa5c99..000000000 --- a/tests/default/indices/aliases/put_alias.yaml +++ /dev/null @@ -1,79 +0,0 @@ -$schema: ../../../../json_schemas/test_story.schema.yaml - -description: Test put_alias variants. -epilogues: - - path: /test_index/_alias/test_alias* - method: DELETE - status: [200, 404] - - path: /test_index - method: DELETE - status: [200, 404] -prologues: - - path: /{index} - method: PUT - parameters: - index: test_index -chapters: - - synopsis: Create an alias - specifying index & alias in path. - path: /{index}/_alias/{name} - method: PUT - parameters: - index: test_index - name: test_alias - response: - status: 200 - payload: - acknowledged: true - - - synopsis: Create an alias - specifying index in path & alias in body. - path: /{index}/_alias - method: PUT - parameters: - index: test_index - request: - payload: - alias: test_alias_1 - response: - status: 200 - payload: - acknowledged: true - - - synopsis: Create an alias - specifying index in body & alias in path. - path: /_alias/{name} - method: PUT - parameters: - name: test_alias_2 - request: - payload: - index: test_index - response: - status: 200 - payload: - acknowledged: true - - - synopsis: Create an alias - specifying index & alias in body. - path: /_alias - method: PUT - request: - payload: - index: test_index - alias: test_alias_3 - response: - status: 200 - payload: - acknowledged: true - - - synopsis: Retrieve aliases. - path: /{index}/_alias - method: GET - parameters: - index: test_index - response: - status: 200 - payload: - test_index: - aliases: - test_alias: {} - test_alias_1: {} - test_alias_2: {} - test_alias_3: {} \ No newline at end of file diff --git a/tests/default/indices/analyze.yaml b/tests/default/indices/analyze.yaml new file mode 100644 index 000000000..9e176a63e --- /dev/null +++ b/tests/default/indices/analyze.yaml @@ -0,0 +1,51 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test /{index}/_analyze. +prologues: + - path: /movies + method: PUT + request: + payload: + settings: + analysis: + analyzer: + default: + type: stop +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Analyze text. + path: /{index}/_analyze + method: POST + parameters: + index: movies + request: + payload: + text: a movie directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: movie + start_offset: 2 + end_offset: 7 + position: 1 + - synopsis: Analyze text specifying an analyzer. + path: /{index}/_analyze + method: GET + parameters: + index: movies + request: + payload: + analyzer: standard + text: a movie directed by Bennett Miller + response: + status: 200 + payload: + tokens: + - token: a + start_offset: 0 + end_offset: 1 + position: 0 diff --git a/tests/default/indices/bulk.yaml b/tests/default/indices/bulk.yaml new file mode 100644 index 000000000..fda3b7554 --- /dev/null +++ b/tests/default/indices/bulk.yaml @@ -0,0 +1,57 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test bulk index endpoint. +epilogues: + - path: /books,movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create an index. + path: /{index}/_bulk + method: POST + parameters: + index: movies + request: + content_type: application/x-ndjson + payload: + - {create: {}} + - {director: Bennett Miller, title: Moneyball, year: 2011} + - synopsis: Delete document in an index. + path: /{index}/_bulk + method: PUT + parameters: + index: movies + request: + content_type: application/x-ndjson + payload: + - {delete: {_id: invalid}} + response: + status: 200 + payload: + errors: false + items: + - delete: + _index: movies + _id: invalid + result: not_found + status: 404 + - synopsis: Bulk index document CRUD. + method: POST + path: /{index}/_bulk + parameters: + index: books + request: + content_type: application/x-ndjson + payload: + - {create: {_id: book_1392214}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {update: {_id: book_1392214}} + - {doc: {pages: 376}} + - {update: {_id: book_1392214}} + - {doc: {pages: 376}, _source: true} + - {update: {_id: book_1392214}} + - {script: {source: ctx._source.pages = 376;}} + - {update: {_id: does_not_exist}} + - {script: {source: 'ctx.op = "none";'}, scripted_upsert: true, upsert: {pages: 375}} + - {delete: {_id: book_1392214}} + From b28c1cfaf4a76d786c789929eaa7fabffd09cb99 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 21 Aug 2024 20:07:41 -0400 Subject: [PATCH 8/8] Add missing /cat API tests. (#521) Signed-off-by: dblock --- tests/default/cat/aliases.yaml | 4 +-- tests/default/cat/allocation.yaml | 12 ++++++++- tests/default/cat/cluster_manager.yaml | 2 +- tests/default/cat/count.yaml | 27 ++++++++++++++++++- tests/default/cat/fielddata.yaml | 15 +++++++++-- tests/default/cat/health.yaml | 22 +++++++-------- tests/default/cat/index.yaml | 2 +- tests/default/cat/indices.yaml | 26 +++++++++++------- tests/default/cat/master.yaml | 2 +- tests/default/cat/nodeattrs.yaml | 2 +- tests/default/cat/nodes.yaml | 4 +-- tests/default/cat/pending_tasks.yaml | 2 +- tests/default/cat/pit_segments/all.yaml | 2 +- .../cat/pit_segments/pit_segments.yaml | 2 +- tests/default/cat/plugins.yaml | 2 +- tests/default/cat/recovery.yaml | 24 ++++++++++++++++- tests/default/cat/repositories.yaml | 12 +++++++++ tests/default/cat/segment_replication.yaml | 20 +++++++++++++- tests/default/cat/segments.yaml | 21 ++++++++++++++- tests/default/cat/shards.yaml | 10 ++++--- tests/default/cat/tasks.yaml | 7 ++++- tests/default/cat/templates.yaml | 5 ++-- tests/default/cat/thread_pool.yaml | 5 ++-- tests/snapshot/cat/repositories.yaml | 2 +- tests/snapshot/cat/snapshots.yaml | 2 +- 25 files changed, 185 insertions(+), 49 deletions(-) create mode 100644 tests/default/cat/repositories.yaml diff --git a/tests/default/cat/aliases.yaml b/tests/default/cat/aliases.yaml index 30f9cebf9..2fb2101d7 100644 --- a/tests/default/cat/aliases.yaml +++ b/tests/default/cat/aliases.yaml @@ -16,13 +16,13 @@ prologues: index: games name: jeux chapters: - - synopsis: Cat with a text response. + - synopsis: List the mapping of aliases to indexes, plus routing and filtering information in plain text. path: /_cat/aliases method: GET response: status: 200 content_type: text/plain - - synopsis: Cat with a json response. + - synopsis: List the mapping of aliases to indexes, plus routing and filtering information in json. path: /_cat/aliases/{name} parameters: format: json diff --git a/tests/default/cat/allocation.yaml b/tests/default/cat/allocation.yaml index 60d603d3b..9d3d854c0 100644 --- a/tests/default/cat/allocation.yaml +++ b/tests/default/cat/allocation.yaml @@ -2,10 +2,20 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/allocation endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List the allocation of disk space for indexes and the number of shards on each node. path: /_cat/allocation method: GET parameters: format: json response: status: 200 + - synopsis: List the allocation of disk space for indexes and the number of shards on a data node. + path: /_cat/allocation/{node_id} + method: GET + parameters: + node_id: data:true + format: json + bytes: kb + local: true + response: + status: 200 diff --git a/tests/default/cat/cluster_manager.yaml b/tests/default/cat/cluster_manager.yaml index e984f2440..d35e9361e 100644 --- a/tests/default/cat/cluster_manager.yaml +++ b/tests/default/cat/cluster_manager.yaml @@ -2,7 +2,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/cluster_manager endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List information that helps identify the elected cluster manager node. version: '>= 2.0' path: /_cat/cluster_manager method: GET diff --git a/tests/default/cat/count.yaml b/tests/default/cat/count.yaml index 31e71c580..37078848d 100644 --- a/tests/default/cat/count.yaml +++ b/tests/default/cat/count.yaml @@ -1,11 +1,36 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/count endpoints. +prologues: + - path: /movies/_doc + method: POST + parameters: + refresh: true + request: + payload: + director: Bennett Miller + title: Moneyball + year: 2011 + status: [201] +epilogues: + - path: /movies + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. + - synopsis: List the number of documents in the cluster. path: /_cat/count method: GET parameters: format: json response: status: 200 + - synopsis: List the number of documents in an index. + path: /_cat/count/{index} + method: GET + parameters: + index: movies + format: json + response: + status: 200 + payload: + - count: '1' diff --git a/tests/default/cat/fielddata.yaml b/tests/default/cat/fielddata.yaml index ca8a11cb5..4afb57de0 100644 --- a/tests/default/cat/fielddata.yaml +++ b/tests/default/cat/fielddata.yaml @@ -2,14 +2,14 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/fielddata endpoints. chapters: - - synopsis: Cat with an empty response. + - synopsis: List the memory size used by each field per node. path: /_cat/fielddata method: GET parameters: format: json response: status: 200 - - synopsis: Cat with a json response (from security-analytics). + - synopsis: List the memory size used by each field per node with data from security-analytics. version: '>= 2.4' path: /_cat/fielddata method: GET @@ -19,3 +19,14 @@ chapters: status: 200 payload: - field: log_types + - synopsis: List the memory size used by each field per node with data from security-analytics. + version: '>= 2.4' + path: /_cat/fielddata/{fields} + method: GET + parameters: + format: json + fields: log_types + response: + status: 200 + payload: + - field: log_types diff --git a/tests/default/cat/health.yaml b/tests/default/cat/health.yaml index dff956b3e..9e9d1df43 100644 --- a/tests/default/cat/health.yaml +++ b/tests/default/cat/health.yaml @@ -2,13 +2,13 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/health endpoints. chapters: - - synopsis: Cat with a default text response. + - synopsis: List the status of the cluster in plain text. method: GET path: /_cat/health response: status: 200 content_type: text/plain - - synopsis: Cat with verbose output (v=true). + - synopsis: List the status of the cluster with verbose output (v=true). method: GET path: /_cat/health parameters: @@ -16,7 +16,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat with headers (h=header1,header2). + - synopsis: List the status of the cluster with headers (h=header1,header2). method: GET path: /_cat/health parameters: @@ -26,7 +26,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat displaying all available headers (help=true). + - synopsis: List the status of the cluster displaying all available headers (help=true). method: GET path: /_cat/health parameters: @@ -34,7 +34,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat with sorted results. + - synopsis: List the status of the cluster with sorted results. method: GET path: /_cat/health parameters: @@ -43,7 +43,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat in different formats (format=json). + - synopsis: List the status of the cluster in different formats (format=json). method: GET path: /_cat/health parameters: @@ -54,7 +54,7 @@ chapters: payload: - node.total: '1' node.data: '1' - - synopsis: Cat with master response (format=json). + - synopsis: List the status of the cluster with master response (format=json). method: GET path: /_cat/health version: < 2.0 @@ -67,7 +67,7 @@ chapters: - node.total: '1' node.data: '1' discovered_master: 'true' - - synopsis: Cat with cluster_manager response (format=json). + - synopsis: List the status of the cluster with cluster_manager response (format=json). method: GET path: /_cat/health version: '>= 2.0' @@ -80,7 +80,7 @@ chapters: - node.total: '1' node.data: '1' discovered_cluster_manager: 'true' - - synopsis: Cat in different formats (format=yaml). + - synopsis: List the status of the cluster in different formats (format=yaml). method: GET path: /_cat/health parameters: @@ -91,7 +91,7 @@ chapters: payload: - node.total: '1' node.data: '1' - - synopsis: Cat in different formats (format=cbor). + - synopsis: List the status of the cluster in different formats (format=cbor). distributions: excluded: - amazon-managed @@ -106,7 +106,7 @@ chapters: payload: - node.total: '1' node.data: '1' - - synopsis: Cat in different formats (format=smile). + - synopsis: List the status of the cluster in different formats (format=smile). distributions: excluded: - amazon-managed diff --git a/tests/default/cat/index.yaml b/tests/default/cat/index.yaml index ae20cdf3a..a40632a60 100644 --- a/tests/default/cat/index.yaml +++ b/tests/default/cat/index.yaml @@ -2,7 +2,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List available compact and aligned text (CAT) APIs in plain text. path: /_cat method: GET response: diff --git a/tests/default/cat/indices.yaml b/tests/default/cat/indices.yaml index 2e2ed1711..2c9e40238 100644 --- a/tests/default/cat/indices.yaml +++ b/tests/default/cat/indices.yaml @@ -13,13 +13,21 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Cat with a default text response. + - synopsis: List information related to indexes (text/plain). method: GET path: /_cat/indices response: status: 200 content_type: text/plain - - synopsis: Cat with verbose output (v=true). + - synopsis: List information related to one index (text/plain). + method: GET + path: /_cat/indices/{index} + parameters: + index: books + response: + status: 200 + content_type: text/plain + - synopsis: List information related to indexes with verbose output (v=true). method: GET path: /_cat/indices parameters: @@ -27,7 +35,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat with headers (h=header1,header2). + - synopsis: List information related to indexes with headers (h=header1,header2). method: GET path: /_cat/indices parameters: @@ -37,7 +45,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat displaying all available headers (help=true). + - synopsis: List information related to indexes displaying all available headers (help=true). method: GET path: /_cat/indices parameters: @@ -45,7 +53,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat with sorted results. + - synopsis: List information related to indexes with sorted results. method: GET path: /_cat/indices parameters: @@ -54,7 +62,7 @@ chapters: response: status: 200 content_type: text/plain - - synopsis: Cat in different formats (format=json). + - synopsis: List information related to indexes in different formats (format=json). method: GET path: /_cat/indices parameters: @@ -62,7 +70,7 @@ chapters: response: status: 200 content_type: application/json - - synopsis: Cat in different formats (format=yaml). + - synopsis: List information related to indexes in different formats (format=yaml). method: GET path: /_cat/indices parameters: @@ -70,7 +78,7 @@ chapters: response: status: 200 content_type: application/yaml - - synopsis: Cat in different formats (format=cbor). + - synopsis: List information related to indexes in different formats (format=cbor). distributions: excluded: - amazon-managed @@ -82,7 +90,7 @@ chapters: response: status: 200 content_type: application/cbor - - synopsis: Cat in different formats (format=smile). + - synopsis: List information related to indexes in different formats (format=smile). distributions: excluded: - amazon-managed diff --git a/tests/default/cat/master.yaml b/tests/default/cat/master.yaml index 93ab62942..ea193902d 100644 --- a/tests/default/cat/master.yaml +++ b/tests/default/cat/master.yaml @@ -3,7 +3,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/master endpoints. version: < 2.0 chapters: - - synopsis: Cat with a json response. + - synopsis: List information that helps identify the elected master node. path: /_cat/master method: GET parameters: diff --git a/tests/default/cat/nodeattrs.yaml b/tests/default/cat/nodeattrs.yaml index f59162b31..1d94a35ca 100644 --- a/tests/default/cat/nodeattrs.yaml +++ b/tests/default/cat/nodeattrs.yaml @@ -6,7 +6,7 @@ distributions: - amazon-managed - amazon-serverless chapters: - - synopsis: Cat with a json response. + - synopsis: List the attributes of custom nodes. path: /_cat/nodeattrs method: GET parameters: diff --git a/tests/default/cat/nodes.yaml b/tests/default/cat/nodes.yaml index 158354e23..5e38cbdaa 100644 --- a/tests/default/cat/nodes.yaml +++ b/tests/default/cat/nodes.yaml @@ -2,7 +2,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/nodes endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List node-level information. path: /_cat/nodes method: GET version: < 2.0 @@ -12,7 +12,7 @@ chapters: status: 200 payload: - master: '*' - - synopsis: Cat with a json response. + - synopsis: List node-level information. path: /_cat/nodes version: '>= 2.0' method: GET diff --git a/tests/default/cat/pending_tasks.yaml b/tests/default/cat/pending_tasks.yaml index c007487ad..811fcbd00 100644 --- a/tests/default/cat/pending_tasks.yaml +++ b/tests/default/cat/pending_tasks.yaml @@ -2,7 +2,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/pending_tasks endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List the progress of all pending tasks. path: /_cat/pending_tasks method: GET parameters: diff --git a/tests/default/cat/pit_segments/all.yaml b/tests/default/cat/pit_segments/all.yaml index 5e6d77632..6bbbdcaae 100644 --- a/tests/default/cat/pit_segments/all.yaml +++ b/tests/default/cat/pit_segments/all.yaml @@ -25,7 +25,7 @@ prologues: output: pit_id: payload.pit_id chapters: - - synopsis: Cat _all with a json response. + - synopsis: List information about the disk utilization of all PITs by describing their Lucene segments. path: /_cat/pit_segments/_all method: GET parameters: diff --git a/tests/default/cat/pit_segments/pit_segments.yaml b/tests/default/cat/pit_segments/pit_segments.yaml index d933ba55b..84b9cafd8 100644 --- a/tests/default/cat/pit_segments/pit_segments.yaml +++ b/tests/default/cat/pit_segments/pit_segments.yaml @@ -25,7 +25,7 @@ prologues: output: pit_id: payload.pit_id chapters: - - synopsis: Cat pit_segments/pit_id with a json response. + - synopsis: List information about the disk utilization of a PIT by describing its Lucene segments. path: /_cat/pit_segments method: GET parameters: diff --git a/tests/default/cat/plugins.yaml b/tests/default/cat/plugins.yaml index a2fb2b42f..a0b6d6b71 100644 --- a/tests/default/cat/plugins.yaml +++ b/tests/default/cat/plugins.yaml @@ -2,7 +2,7 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/plugins endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List the names, components, and versions of the installed plugins. path: /_cat/plugins method: GET parameters: diff --git a/tests/default/cat/recovery.yaml b/tests/default/cat/recovery.yaml index a1341f219..1731e42b4 100644 --- a/tests/default/cat/recovery.yaml +++ b/tests/default/cat/recovery.yaml @@ -1,11 +1,33 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/recovery endpoints. +prologues: + - path: /{index} + method: PUT + parameters: + index: books +epilogues: + - path: /books + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. + - synopsis: List all completed and ongoing index and shard recoveries. path: /_cat/recovery method: GET parameters: format: json response: status: 200 + - synopsis: List all completed and ongoing index and shard recoveries for a given index. + path: /_cat/recovery/{index} + method: GET + parameters: + format: json + index: + - books + active_only: true + bytes: kb + detailed: true + time: m + response: + status: 200 diff --git a/tests/default/cat/repositories.yaml b/tests/default/cat/repositories.yaml new file mode 100644 index 000000000..bf113f552 --- /dev/null +++ b/tests/default/cat/repositories.yaml @@ -0,0 +1,12 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test cat/repositories endpoints. +chapters: + - synopsis: List all snapshot repositories for a cluster. + path: /_cat/repositories + method: GET + parameters: + format: json + local: true + response: + status: 200 diff --git a/tests/default/cat/segment_replication.yaml b/tests/default/cat/segment_replication.yaml index 0495ece5f..9d8d5002a 100644 --- a/tests/default/cat/segment_replication.yaml +++ b/tests/default/cat/segment_replication.yaml @@ -2,11 +2,29 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/segment_replication endpoints. version: '>= 2.7' +prologues: + - path: /{index} + method: PUT + parameters: + index: books +epilogues: + - path: /books + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. + - synopsis: List information about active and last completed segment replication events on each replica shard. path: /_cat/segment_replication method: GET parameters: format: json response: status: 200 + - synopsis: List information about active and last completed segment replication events on each replica shard for an index. + path: /_cat/segment_replication/{index} + method: GET + parameters: + format: json + index: + - books + response: + status: 200 diff --git a/tests/default/cat/segments.yaml b/tests/default/cat/segments.yaml index b8cbb8f56..ecae37d0f 100644 --- a/tests/default/cat/segments.yaml +++ b/tests/default/cat/segments.yaml @@ -1,11 +1,30 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/segments endpoints. +prologues: + - path: /{index} + method: PUT + parameters: + index: books +epilogues: + - path: /books + method: DELETE + status: [200, 404] chapters: - - synopsis: Cat with a json response. + - synopsis: List Lucene segment-level information for all indices. path: /_cat/segments method: GET parameters: format: json response: status: 200 + - synopsis: List Lucene segment-level information for an index. + path: /_cat/segments/{index} + method: GET + parameters: + format: json + bytes: kb + index: + - books + response: + status: 200 diff --git a/tests/default/cat/shards.yaml b/tests/default/cat/shards.yaml index f2ab5c147..033f55ff8 100644 --- a/tests/default/cat/shards.yaml +++ b/tests/default/cat/shards.yaml @@ -11,17 +11,21 @@ prologues: parameters: index: games chapters: - - synopsis: Cat all with a json response. + - synopsis: List the state of all primary and replica shards and how they are distributed. path: /_cat/shards method: GET parameters: format: json - - synopsis: Cat index shards with a json response. + - synopsis: List the state of all primary and replica shards and how they are distributed for an index. path: /_cat/shards/{index} method: GET parameters: format: json - index: games + index: + - games + bytes: kb + local: true + time: s response: status: 200 payload: diff --git a/tests/default/cat/tasks.yaml b/tests/default/cat/tasks.yaml index aedcaab42..683652769 100644 --- a/tests/default/cat/tasks.yaml +++ b/tests/default/cat/tasks.yaml @@ -2,10 +2,15 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/tasks endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List the progress of all tasks currently running on your cluster. path: /_cat/tasks method: GET parameters: format: json + nodes: + - local + detailed: true + parent_task_id: node_id:-1 + time: s response: status: 200 diff --git a/tests/default/cat/templates.yaml b/tests/default/cat/templates.yaml index 330b6c12c..4c6b2e51e 100644 --- a/tests/default/cat/templates.yaml +++ b/tests/default/cat/templates.yaml @@ -18,19 +18,20 @@ prologues: number_of_shards: 2 number_of_replicas: 2 chapters: - - synopsis: Cat with a json response. + - synopsis: List the names, patterns, order numbers, and version numbers of index templates. path: /_cat/templates method: GET parameters: format: json response: status: 200 - - synopsis: Cat a specific template with a json response. + - synopsis: List the names, patterns, order numbers, and version numbers of index templates. path: /_cat/templates/{name} method: GET parameters: format: json name: daily_logs + local: true response: status: 200 payload: diff --git a/tests/default/cat/thread_pool.yaml b/tests/default/cat/thread_pool.yaml index b38cd8772..59a211ebf 100644 --- a/tests/default/cat/thread_pool.yaml +++ b/tests/default/cat/thread_pool.yaml @@ -2,19 +2,20 @@ $schema: ../../../json_schemas/test_story.schema.yaml description: Test cat/thread_pool endpoints. chapters: - - synopsis: Cat with a json response. + - synopsis: List the active, queued, and rejected threads of different thread pools on each node. path: /_cat/thread_pool method: GET parameters: format: json response: status: 200 - - synopsis: Cat of a specific thread pool with a json response. + - synopsis: List the active, queued, and rejected threads of different thread pools on each node. path: /_cat/thread_pool/{thread_pool_patterns} method: GET parameters: format: json thread_pool_patterns: generic + local: true response: status: 200 payload: diff --git a/tests/snapshot/cat/repositories.yaml b/tests/snapshot/cat/repositories.yaml index 89d258e23..3124a0172 100644 --- a/tests/snapshot/cat/repositories.yaml +++ b/tests/snapshot/cat/repositories.yaml @@ -18,7 +18,7 @@ prologues: settings: location: /tmp/opensearch/repo chapters: - - synopsis: Cat with a json response. + - synopsis: List all snapshot repositories for a cluster. path: /_cat/repositories method: GET parameters: diff --git a/tests/snapshot/cat/snapshots.yaml b/tests/snapshot/cat/snapshots.yaml index 1e35ea574..81a51bfc2 100644 --- a/tests/snapshot/cat/snapshots.yaml +++ b/tests/snapshot/cat/snapshots.yaml @@ -14,7 +14,7 @@ prologues: settings: location: /tmp/opensearch/repo chapters: - - synopsis: Cat with a json response. + - synopsis: List all snapshots for a repository. path: /_cat/snapshots/{repository} method: GET parameters: