From 5149b66fc9e6543ea6185a9734352c9656c84412 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Dec 2024 13:46:37 -0500 Subject: [PATCH 1/6] Added tests for PUT /_settings. (#742) Signed-off-by: dblock --- tests/plugins/security/_core/settings.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/plugins/security/_core/settings.yaml diff --git a/tests/plugins/security/_core/settings.yaml b/tests/plugins/security/_core/settings.yaml new file mode 100644 index 00000000..743bc4a4 --- /dev/null +++ b/tests/plugins/security/_core/settings.yaml @@ -0,0 +1,15 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test settings. +chapters: + - synopsis: Update cluster settings. + path: /_settings + method: PUT + request: + payload: + settings: + index: + number_of_replicas: 4 + - synopsis: Get settings. + path: /_settings + method: GET From a631452252e1f45816ba084f1620296dd9a68658 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Dec 2024 13:56:25 -0500 Subject: [PATCH 2/6] Added tests for GET /_stats/{metric}. (#743) Signed-off-by: dblock --- tests/default/_core/stats.yaml | 25 +++++++++++++++++++++++++ tests/default/indices/stats.yaml | 12 ------------ 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 tests/default/_core/stats.yaml diff --git a/tests/default/_core/stats.yaml b/tests/default/_core/stats.yaml new file mode 100644 index 00000000..98d898ce --- /dev/null +++ b/tests/default/_core/stats.yaml @@ -0,0 +1,25 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test stats. +chapters: + - synopsis: Get global stats. + path: /_stats + method: GET + response: + status: 200 + - synopsis: Get global stats with human fields. + path: /_stats + method: GET + parameters: + human: true + response: + status: 200 + - synopsis: Get stats for a given metric. + path: /_stats/{metric} + method: GET + parameters: + metric: docs + expand_wildcards: all + forbid_closed_indices: false + groups: '*' + level: indices diff --git a/tests/default/indices/stats.yaml b/tests/default/indices/stats.yaml index 24caa7e2..32e881b3 100644 --- a/tests/default/indices/stats.yaml +++ b/tests/default/indices/stats.yaml @@ -9,18 +9,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get global stats. - path: /_stats - method: GET - response: - status: 200 - - synopsis: Get global stats with human fields. - path: /_stats - method: GET - parameters: - human: true - response: - status: 200 - synopsis: Get stats for an index. path: /{index}/_stats method: GET From dc41b129332190fdb50e09bf708ac369dbc9bea9 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 18 Dec 2024 01:19:45 -0500 Subject: [PATCH 3/6] Added missing task tests. (#746) Signed-off-by: dblock --- tests/default/_core/tasks.yaml | 39 +++++++++++++++++++++++++++ tests/default/_core/tasks/cancel.yaml | 33 +++++++++++++++++++++++ tests/default/tasks/list.yaml | 25 ----------------- 3 files changed, 72 insertions(+), 25 deletions(-) create mode 100644 tests/default/_core/tasks.yaml create mode 100644 tests/default/_core/tasks/cancel.yaml delete mode 100644 tests/default/tasks/list.yaml diff --git a/tests/default/_core/tasks.yaml b/tests/default/_core/tasks.yaml new file mode 100644 index 00000000..3c40bd60 --- /dev/null +++ b/tests/default/_core/tasks.yaml @@ -0,0 +1,39 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test tasks endpoint. +prologues: + - path: /books/_delete_by_query + id: task + method: POST + parameters: + refresh: true + conflicts: proceed + wait_for_completion: false + request: + payload: + query: + match_all: {} + output: + id: payload.task +chapters: + - synopsis: List tasks grouped by node. + path: /_tasks + method: GET + parameters: + group_by: nodes + - synopsis: List tasks grouped by parent. + path: /_tasks + method: GET + parameters: + group_by: parents + - synopsis: List tasks grouped by none. + path: /_tasks + method: GET + parameters: + group_by: none + - synopsis: Get task by id. + id: task + path: /_tasks/{task_id} + method: GET + parameters: + task_id: ${task.id} diff --git a/tests/default/_core/tasks/cancel.yaml b/tests/default/_core/tasks/cancel.yaml new file mode 100644 index 00000000..d1100f78 --- /dev/null +++ b/tests/default/_core/tasks/cancel.yaml @@ -0,0 +1,33 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test tasks endpoint. +prologues: + - path: /books/_delete_by_query + id: task + method: POST + parameters: + refresh: true + conflicts: proceed + wait_for_completion: false + request: + payload: + query: + match_all: {} + output: + id: payload.task +chapters: + - synopsis: Cancel all tasks. + path: /_tasks/_cancel + method: POST + - synopsis: Cancel a task. + path: /_tasks/{task_id}/_cancel + method: POST + parameters: + task_id: ${task.id} + response: + status: 200 + payload: + node_failures: + - type: failed_node_exception + caused_by: + type: resource_not_found_exception diff --git a/tests/default/tasks/list.yaml b/tests/default/tasks/list.yaml deleted file mode 100644 index a687e170..00000000 --- a/tests/default/tasks/list.yaml +++ /dev/null @@ -1,25 +0,0 @@ -$schema: ../../../json_schemas/test_story.schema.yaml - -description: Test tasks list endpoint. -chapters: - - synopsis: List tasks grouped by node. - path: /_tasks - method: GET - parameters: - group_by: nodes - response: - status: 200 - - synopsis: List tasks grouped by parent. - path: /_tasks - method: GET - parameters: - group_by: parents - response: - status: 200 - - synopsis: List tasks grouped by none. - path: /_tasks - method: GET - parameters: - group_by: none - response: - status: 200 \ No newline at end of file From 4c03b599ed511f68b8ac822cbb12a9b64e5b0810 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 18 Dec 2024 01:22:07 -0500 Subject: [PATCH 4/6] Added missing wlm tests. (#744) Signed-off-by: dblock --- .../workload-management/query_group.yaml | 67 ------------------- .../workload-management/wlm/query_group.yaml | 41 ++++++++++++ 2 files changed, 41 insertions(+), 67 deletions(-) delete mode 100644 tests/plugins/workload-management/query_group.yaml create mode 100644 tests/plugins/workload-management/wlm/query_group.yaml diff --git a/tests/plugins/workload-management/query_group.yaml b/tests/plugins/workload-management/query_group.yaml deleted file mode 100644 index 6f2b8636..00000000 --- a/tests/plugins/workload-management/query_group.yaml +++ /dev/null @@ -1,67 +0,0 @@ -$schema: ../../../json_schemas/test_story.schema.yaml - -description: This story tests all endpoints relevant to QueryGroups, from creation to deletion. - -prologues: - - path: /_wlm/query_group - version: '>= 2.17' - method: PUT - request: - payload: - name: analytics - resource_limits: - memory: 0.1 - resiliency_mode: monitor - status: [200] - - path: /_wlm/query_group - version: '>= 2.17' - method: PUT - request: - payload: - name: analytics2 - resource_limits: - memory: 0.15 - cpu: 0.05 - resiliency_mode: monitor - status: [200] - -chapters: - - synopsis: Update the QueryGroup named `analytics2`, with resiliency_mode to be enforced and memory to be 0.25 and cpu to be 0.25. - version: '>= 2.17' - path: /_wlm/query_group/{name} - method: PUT - parameters: - name: analytics2 - request: - payload: - resource_limits: - memory: 0.25 - cpu: 0.25 - resiliency_mode: enforced - response: - status: 200 - - synopsis: Get the QueryGroup named `analytics2`. - version: '>= 2.17' - path: /_wlm/query_group/{name} - method: GET - parameters: - name: analytics2 - response: - status: 200 - payload: - query_groups: - - name: analytics2 - resource_limits: - memory: 0.25 - cpu: 0.25 - resiliency_mode: enforced - -epilogues: - - path: /_wlm/query_group/analytics - version: '>= 2.17' - method: DELETE - status: [200] - - path: /_wlm/query_group/analytics2 - version: '>= 2.17' - method: DELETE - status: [200] diff --git a/tests/plugins/workload-management/wlm/query_group.yaml b/tests/plugins/workload-management/wlm/query_group.yaml new file mode 100644 index 00000000..aa6a68b9 --- /dev/null +++ b/tests/plugins/workload-management/wlm/query_group.yaml @@ -0,0 +1,41 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: This story tests all endpoints relevant to QueryGroups, from creation to deletion. +version: '>= 2.17' + +chapters: + - synopsis: Create a QueryGroup. + path: /_wlm/query_group + method: PUT + request: + payload: + name: analytics + resource_limits: + memory: 0.1 + resiliency_mode: monitor + - synopsis: Get the QueryGroup by name. + path: /_wlm/query_group/{name} + method: GET + parameters: + name: analytics + response: + status: 200 + payload: + query_groups: + - name: analytics + resource_limits: + memory: 0.1 + resiliency_mode: monitor + - synopsis: Get all QueryGroups. + path: /_wlm/query_group + method: GET + - synopsis: Delete a QueryGroup. + path: /_wlm/query_group/{name} + parameters: + name: analytics + method: DELETE + +epilogues: + - path: /_wlm/query_group/analytics + method: DELETE + status: [200, 404] From 7a3241996964e6429810660e7cb78cf32e1d7829 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 18 Dec 2024 11:39:55 -0500 Subject: [PATCH 5/6] Added tests for POST /_remotestore/_restore. (#741) * Added tests for POST /_remotestore/_restore. Signed-off-by: dblock * Fixed book and movie years. Signed-off-by: dblock --------- Signed-off-by: dblock --- .github/workflows/test-spec.yml | 2 ++ tests/default/_core/mget.yaml | 2 +- tests/default/_core/msearch/index.yaml | 2 +- tests/default/_core/reindex/pipeline.yaml | 2 +- tests/default/cat/pending_tasks.yaml | 2 +- tests/default/cluster/pending_tasks.yaml | 2 +- tests/default/indices/delete_by_query.yaml | 4 +-- .../indices/delete_by_query/rethrottle.yaml | 4 +-- tests/default/indices/msearch/index.yaml | 2 +- tests/default/transforms/preview.yaml | 2 +- tests/default/transforms/transform.yaml | 2 +- .../default/transforms/transform/explain.yaml | 2 +- tests/default/transforms/transform/start.yaml | 2 +- tests/default/transforms/transform/stop.yaml | 2 +- tests/remote_store/docker-compose.yml | 23 ++++++++++++++++ tests/remote_store/remote_store/restore.yaml | 26 +++++++++++++++++++ 16 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 tests/remote_store/docker-compose.yml create mode 100644 tests/remote_store/remote_store/restore.yaml diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index d3b495ca..d166b375 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -37,6 +37,8 @@ jobs: tests: routing - version: 2.18.0 tests: snapshot + - version: 2.18.0 + tests: remote_store - version: 2.18.0 tests: dangling url: http://localhost:9200 diff --git a/tests/default/_core/mget.yaml b/tests/default/_core/mget.yaml index 0d409ecf..ee00dd57 100644 --- a/tests/default/_core/mget.yaml +++ b/tests/default/_core/mget.yaml @@ -17,7 +17,7 @@ prologues: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book1}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: movies, _id: movie1}} - {director: Bennett Miller, title: The Cruise, year: 1998} - {create: {_index: movies, _id: movie2}} diff --git a/tests/default/_core/msearch/index.yaml b/tests/default/_core/msearch/index.yaml index 7cb89340..1a3a3df7 100644 --- a/tests/default/_core/msearch/index.yaml +++ b/tests/default/_core/msearch/index.yaml @@ -17,7 +17,7 @@ prologues: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book1}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: movies, _id: movie1}} - {director: Bennett Miller, title: The Cruise, year: 1998} - {create: {_index: movies, _id: movie2}} diff --git a/tests/default/_core/reindex/pipeline.yaml b/tests/default/_core/reindex/pipeline.yaml index 30a2fd6a..a91ecc8a 100644 --- a/tests/default/_core/reindex/pipeline.yaml +++ b/tests/default/_core/reindex/pipeline.yaml @@ -40,7 +40,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] chapters: - synopsis: Transform documents using a pipeline. diff --git a/tests/default/cat/pending_tasks.yaml b/tests/default/cat/pending_tasks.yaml index 4a59f67e..67538614 100644 --- a/tests/default/cat/pending_tasks.yaml +++ b/tests/default/cat/pending_tasks.yaml @@ -17,7 +17,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] - path: /_reindex method: POST diff --git a/tests/default/cluster/pending_tasks.yaml b/tests/default/cluster/pending_tasks.yaml index f595fce7..4a0e0781 100644 --- a/tests/default/cluster/pending_tasks.yaml +++ b/tests/default/cluster/pending_tasks.yaml @@ -17,7 +17,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] - path: /_reindex method: POST diff --git a/tests/default/indices/delete_by_query.yaml b/tests/default/indices/delete_by_query.yaml index b6c9c0a9..4c518fd0 100644 --- a/tests/default/indices/delete_by_query.yaml +++ b/tests/default/indices/delete_by_query.yaml @@ -14,9 +14,9 @@ prologues: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: books, _id: book_1392215}} - - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 91} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} chapters: - synopsis: Delete documents in the index (full query term, script). path: /{index}/_delete_by_query diff --git a/tests/default/indices/delete_by_query/rethrottle.yaml b/tests/default/indices/delete_by_query/rethrottle.yaml index 3169ab20..1e1293ab 100644 --- a/tests/default/indices/delete_by_query/rethrottle.yaml +++ b/tests/default/indices/delete_by_query/rethrottle.yaml @@ -14,9 +14,9 @@ prologues: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book_1392214}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: books, _id: book_1392215}} - - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 91} + - {author: Elizabeth Rudnick, title: Beauty and the Beast, year: 1991} - path: /books/_delete_by_query id: task method: POST diff --git a/tests/default/indices/msearch/index.yaml b/tests/default/indices/msearch/index.yaml index 53fbb712..ac856056 100644 --- a/tests/default/indices/msearch/index.yaml +++ b/tests/default/indices/msearch/index.yaml @@ -10,7 +10,7 @@ prologues: content_type: application/x-ndjson payload: - {create: {_index: books, _id: book1}} - - {author: Harper Lee, title: To Kill a Mockingbird, year: 60} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - {create: {_index: books, _id: book2}} - {director: Bennett Miller, title: The Cruise, year: 1998} - {create: {_index: books, _id: book3}} diff --git a/tests/default/transforms/preview.yaml b/tests/default/transforms/preview.yaml index f26becd9..1624e78d 100644 --- a/tests/default/transforms/preview.yaml +++ b/tests/default/transforms/preview.yaml @@ -10,7 +10,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] epilogues: - path: /movies diff --git a/tests/default/transforms/transform.yaml b/tests/default/transforms/transform.yaml index eaae0a83..fb53e422 100644 --- a/tests/default/transforms/transform.yaml +++ b/tests/default/transforms/transform.yaml @@ -10,7 +10,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] epilogues: - path: /_plugins/_transform/movies-to-films diff --git a/tests/default/transforms/transform/explain.yaml b/tests/default/transforms/transform/explain.yaml index 7adc70c4..028b6d5b 100644 --- a/tests/default/transforms/transform/explain.yaml +++ b/tests/default/transforms/transform/explain.yaml @@ -10,7 +10,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] - path: /_plugins/_transform/movies-to-films method: PUT diff --git a/tests/default/transforms/transform/start.yaml b/tests/default/transforms/transform/start.yaml index 55d681bc..dba5678e 100644 --- a/tests/default/transforms/transform/start.yaml +++ b/tests/default/transforms/transform/start.yaml @@ -10,7 +10,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] - path: /_plugins/_transform/movies-to-films method: PUT diff --git a/tests/default/transforms/transform/stop.yaml b/tests/default/transforms/transform/stop.yaml index 7f3f9276..7671b35f 100644 --- a/tests/default/transforms/transform/stop.yaml +++ b/tests/default/transforms/transform/stop.yaml @@ -10,7 +10,7 @@ prologues: request: payload: title: Beauty and the Beast - year: 91 + year: 1991 status: [201] - path: /_plugins/_transform/movies-to-films method: PUT diff --git a/tests/remote_store/docker-compose.yml b/tests/remote_store/docker-compose.yml new file mode 100644 index 00000000..89c151c7 --- /dev/null +++ b/tests/remote_store/docker-compose.yml @@ -0,0 +1,23 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - cluster.remote_store.state.enabled=true + - discovery.type=single-node + - node.attr.remote_store.repository.fs-segment-store.settings.location=/tmp/opensearch/repo/segment + - node.attr.remote_store.repository.fs-segment-store.type=fs + - node.attr.remote_store.repository.fs-state-store.settings.location=/tmp/opensearch/repo/state + - node.attr.remote_store.repository.fs-state-store.type=fs + - node.attr.remote_store.repository.fs-translog-store.settings.location=/tmp/opensearch/repo/translog + - node.attr.remote_store.repository.fs-translog-store.type=fs + - node.attr.remote_store.segment.repository=fs-segment-store + - node.attr.remote_store.state.repository=fs-state-store + - node.attr.remote_store.translog.repository=fs-translog-store + - path.repo=/tmp/opensearch/repo diff --git a/tests/remote_store/remote_store/restore.yaml b/tests/remote_store/remote_store/restore.yaml new file mode 100644 index 00000000..a15d6bf5 --- /dev/null +++ b/tests/remote_store/remote_store/restore.yaml @@ -0,0 +1,26 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test restore from a remote store. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books, _id: book1}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} + - {create: {_index: movies, _id: movie1}} + - {director: Bennett Miller, title: The Cruise, year: 1998} + - {create: {_index: movies, _id: movie2}} + - {director: Nicolas Winding Refn, title: Drive, year: 1960} +chapters: + - synopsis: Restore from remote store. + path: /_remotestore/_restore + method: POST + request: + payload: + indices: + - movies + From 1b6935fbf139ec5d88690bc338b069e877857f9f Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Wed, 18 Dec 2024 11:50:28 -0500 Subject: [PATCH 6/6] Added missing cancellation_time_millis. (#747) Signed-off-by: dblock --- CHANGELOG.md | 1 + spec/schemas/tasks._common.yaml | 2 ++ tests/default/_core/tasks/cancel.yaml | 20 +++++++++++++------- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 956b2fbc..60c0eb3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) - Added `_type` to `termvector` and `mtermvector` ([#734](https://github.com/opensearch-project/opensearch-api-specification/pull/734)) - Added missing `cancelled` and `resource_stats` to `/_reindex/{task_id}/_rethrottle` ([#740](https://github.com/opensearch-project/opensearch-api-specification/pull/740)) +- Added missing `cancellation_time_millis` to `POST /_tasks/_cancel` ([#747](https://github.com/opensearch-project/opensearch-api-specification/pull/747)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/schemas/tasks._common.yaml b/spec/schemas/tasks._common.yaml index 7c165258..cf6c8527 100644 --- a/spec/schemas/tasks._common.yaml +++ b/spec/schemas/tasks._common.yaml @@ -44,6 +44,8 @@ components: type: boolean cancellable: type: boolean + cancellation_time_millis: + $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' description: type: string headers: diff --git a/tests/default/_core/tasks/cancel.yaml b/tests/default/_core/tasks/cancel.yaml index d1100f78..a024f4ff 100644 --- a/tests/default/_core/tasks/cancel.yaml +++ b/tests/default/_core/tasks/cancel.yaml @@ -1,7 +1,20 @@ $schema: ../../../../json_schemas/test_story.schema.yaml description: Test tasks endpoint. +epilogues: + - path: /books + method: DELETE + status: [200, 404] prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: books}} + - {author: Harper Lee, title: To Kill a Mockingbird, year: 1960} - path: /books/_delete_by_query id: task method: POST @@ -24,10 +37,3 @@ chapters: method: POST parameters: task_id: ${task.id} - response: - status: 200 - payload: - node_failures: - - type: failed_node_exception - caused_by: - type: resource_not_found_exception