[SIEM][Detection Engine] Fixes critical blocker where signals on sign…

…als are not operating (#55479)

## Summary

This fixes halting, infinite creation of signals, and cyclic issues with signals when they are reflected on their own index. Without this fix, you could get a user who looks back at a signals index as both their input and output index and forever generates new signals forever and ever and ever until the heath death of the universe. 

* Changes the data structure to support parent and ancestors
* Adds a check for the parent and ancestors
* Adds README.md and in-depth testing of cyclic concepts
* Adds README.md and in-depth testing of depth levels of signal concepts
* Added unit tests for both use cases
* Removed extra console.log statement found in the code base

Follow the two README.md's included for testing and explanation of how it works.

See `test_cases/signals_on_signals/depth_test`
See `test_cases/signals_on_signals/halting_test`

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~

- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)

x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_rules.js

@@ -114,7 +114,6 @@ async function main() {
         );
         return [...accum, parsedLine];
       } catch (err) {
-        console.log('error parsing a line in this file:', json, line);
         return accum;
       }
     }, []);

x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md

@@ -94,7 +94,7 @@ You should see the new rules created like so:
       "interval": "5m",
       "rule_id": "rule-1",
       "language": "kuery",
-      "output_index": ".siem-signals-frank-hassanabad",
+      "output_index": ".siem-signals-some-name",
       "max_signals": 100,
       "risk_score": 1,
       "name": "Detect Root/Admin Users",

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -371,7 +371,7 @@ export const getMockPrivileges = () => ({
     create_snapshot: true,
   },
   index: {
-    '.siem-signals-frank-hassanabad-test-space': {
+    '.siem-signals-test-space': {
       all: false,
       manage_ilm: true,
       read: false,

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/signals_mapping.json

@@ -5,6 +5,9 @@
         "properties": {
           "parent": {
             "properties": {
+              "rule": {
+                "type": "keyword"
+              },
               "index": {
                 "type": "keyword"
               },
@@ -19,6 +22,9 @@
               }
             }
           },
+          "ancestors": {
+            "type": "object"
+          },
           "rule": {
             "properties": {
               "id": {

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/import/multiple_ruleid_queries.ndjson

@@ -1,3 +1,3 @@
-{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
-{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
+{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
+{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
 {"exported_count":2,"missing_rules":[],"missing_rules_count":0}

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/README.md

@@ -4,15 +4,3 @@ use these type of rule based messages when writing pure REST API calls. These me
 more of what you would see "behind the scenes" when you are using Kibana UI which can
 create rules with additional "meta" data or other implementation details that aren't really
 a concern for a regular REST API user.
-
-To post all of them to see in the UI, with the scripts folder as your current working directory:
-
-```sh
-./post_rule.sh ./rules/test_cases/*.json
-```
-
-To post only one at a time:
-
-```sh
-./post_rule.sh ./rules/test_cases/<filename>.json
-```

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/multiple_ruleid_queries_corrupted.ndjson → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/imports/multiple_ruleid_queries_corrupted.ndjson

@@ -1,4 +1,4 @@
-{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1},
-{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
-{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-3","language":"kuery","output_index":".siem-signals-frank-hassanabad-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
+{"created_at":"2020-01-09T01:38:00.740Z","updated_at":"2020-01-09T01:38:00.740Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"6688f367-1aa2-4895-a5a8-b3701eecf57d","immutable":false,"interval":"5m","rule_id":"query-rule-id-1","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":1,"name":"Query with a rule id Number 1","query":"user.name: root or user.name: admin","references":[],"severity":"high","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1},
+{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-2","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
+{"created_at":"2020-01-09T01:38:00.745Z","updated_at":"2020-01-09T01:38:00.745Z","created_by":"elastic_kibana","description":"Query with a rule_id that acts like an external id","enabled":true,"false_positives":[],"from":"now-6m","id":"7a912444-6cfa-4c8f-83f4-2b26fb2a2ed9","immutable":false,"interval":"5m","rule_id":"query-rule-id-3","language":"kuery","output_index":".siem-signals-default","max_signals":100,"risk_score":2,"name":"Query with a rule id Number 2","query":"user.name: root or user.name: admin","references":[],"severity":"low","updated_by":"elastic_kibana","tags":[],"to":"now","type":"query","threats":[],"version":1}
 {"exported_count":2,"missing_rules":[],"missing_rules_count":0}